forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-26 06:36:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

core, eth/filter: optimize log retrieval

Browse source 
This PR includes:

- optimizes (re)generation of mipmap bloom pre-images
- optimizes receipts integration on fast sync
- optimizes mipmap bloom levels for logs searching
- replaces custom bloom filter implementation with package github.com/tylertreat/BoomFilters

use db batches for inserting receipts during fast sync
This commit is contained in:
Bas van Kervel 2017-01-12 09:58:28 +01:00
parent 296450451b
commit 34445d22cf
28 changed files with 3977 additions and 356 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

164
core/blockchain.go
View file

@ -23,7 +23,6 @@ import (
"io"
"math/big"
mrand "math/rand"
"runtime"
"sync"
"sync/atomic"
"time"
@ -44,6 +43,7 @@ import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/hashicorp/golang-lru"
boom "github.com/tylertreat/BoomFilters"
)
var (
@ -691,102 +691,79 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain
self.wg.Add(1)
defer self.wg.Done()
// Collect some import statistics to report on
stats := struct{ processed, ignored int32 }{}
start := time.Now()
var (
chainLength = len(blockChain)
stats = struct{ processed, ignored int }{}
start = time.Now()
insertedReceipts = make(map[uint64]types.Receipts)
)
// Create the block importing task queue and worker functions
tasks := make(chan int, len(blockChain))
for i := 0; i < len(blockChain) && i < len(receiptChain); i++ {
tasks <- i
if len(receiptChain) < chainLength {
chainLength = len(receiptChain)
}
close(tasks)
errs, failed := make([]error, len(tasks)), int32(0)
process := func(worker int) {
for index := range tasks {
block, receipts := blockChain[index], receiptChain[index]
for i := 0; i < chainLength; i++ {
batch := self.chainDb.NewBatch()
block, receipts := blockChain[i], receiptChain[i]
// Short circuit insertion if shutting down or processing failed
if atomic.LoadInt32(&self.procInterrupt) == 1 {
return
}
if atomic.LoadInt32(&failed) > 0 {
return
return 0, nil
}
// Short circuit if the owner header is unknown
if !self.HasHeader(block.Hash()) {
errs[index] = fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4])
atomic.AddInt32(&failed, 1)
return
return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4])
}
// Skip if the entire data is already known
if self.HasBlock(block.Hash()) {
atomic.AddInt32(&stats.ignored, 1)
stats.ignored += len(receipts)
continue
}
// Compute all the non-consensus fields of the receipts
SetReceiptsData(self.config, block, receipts)
// Write all the data out into the database
if err := WriteBody(self.chainDb, block.Hash(), block.NumberU64(), block.Body()); err != nil {
errs[index] = fmt.Errorf("failed to write block body: %v", err)
atomic.AddInt32(&failed, 1)
glog.Fatal(errs[index])
return
}
if err := WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil {
errs[index] = fmt.Errorf("failed to write block receipts: %v", err)
atomic.AddInt32(&failed, 1)
glog.Fatal(errs[index])
return
}
if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
errs[index] = fmt.Errorf("failed to write log blooms: %v", err)
atomic.AddInt32(&failed, 1)
glog.Fatal(errs[index])
return
}
if err := WriteTransactions(self.chainDb, block); err != nil {
errs[index] = fmt.Errorf("failed to write individual transactions: %v", err)
atomic.AddInt32(&failed, 1)
glog.Fatal(errs[index])
return
}
if err := WriteReceipts(self.chainDb, receipts); err != nil {
errs[index] = fmt.Errorf("failed to write individual receipts: %v", err)
atomic.AddInt32(&failed, 1)
glog.Fatal(errs[index])
return
}
atomic.AddInt32(&stats.processed, 1)
}
}
// Start as many worker threads as goroutines allowed
pending := new(sync.WaitGroup)
for i := 0; i < runtime.GOMAXPROCS(0); i++ {
pending.Add(1)
go func(id int) {
defer pending.Done()
process(id)
}(i)
}
pending.Wait()
// If anything failed, report
if failed > 0 {
for i, err := range errs {
if err != nil {
return i, err
// Write all the data out into a batch
if err := WriteBodyBatch(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil {
err := fmt.Errorf("failed to write block body: %v", err)
glog.Fatal(err)
}
if err := WriteBlockReceiptsBatch(batch, block.Hash(), block.NumberU64(), receipts); err != nil {
err := fmt.Errorf("failed to write block receipts: %v", err)
glog.Fatal(err)
}
if err := WriteTransactionsBatch(batch, block); err != nil {
err := fmt.Errorf("failed to write individual transactions: %v", err)
glog.Fatal(err)
}
if err := WriteReceiptsBatch(batch, receipts); err != nil {
err := fmt.Errorf("failed to write individual receipts: %v", err)
glog.Fatal(err)
}
// commit batch to db.
if err := batch.Write(); err != nil {
glog.Fatalf("failed to write receipts chain into database: %v", err)
}
insertedReceipts[block.NumberU64()] = receipts
stats.processed += len(receipts)
}
if err := updateReceiptAddressBlooms(self.chainDb, insertedReceipts); err != nil {
glog.Fatalf("failed to write log address index to database: %v", err)
}
if atomic.LoadInt32(&self.procInterrupt) == 1 {
glog.V(logger.Debug).Infoln("premature abort during receipt chain processing")
return 0, nil
}
// Update the head fast sync block if better
self.mu.Lock()
head := blockChain[len(errs)-1]
head := blockChain[chainLength-1]
if self.GetTd(self.currentFastBlock.Hash(), self.currentFastBlock.NumberU64()).Cmp(self.GetTd(head.Hash(), head.NumberU64())) < 0 {
if err := WriteHeadFastBlockHash(self.chainDb, head.Hash()); err != nil {
glog.Fatalf("failed to update head fast block hash: %v", err)
@ -796,17 +773,54 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain
self.mu.Unlock()
// Report some public statistics so the user has a clue what's going on
first, last := blockChain[0], blockChain[len(blockChain)-1]
first, last := blockChain[0], blockChain[chainLength-1]
ignored := ""
ignoredStr := ""
if stats.ignored > 0 {
ignored = fmt.Sprintf(" (%d ignored)", stats.ignored)
ignoredStr = fmt.Sprintf(" (%d ignored)", stats.ignored)
}
glog.V(logger.Info).Infof("imported %4d receipts in %9v. #%d [%x… / %x…]%s", stats.processed, common.PrettyDuration(time.Since(start)), last.Number(), first.Hash().Bytes()[:4], last.Hash().Bytes()[:4], ignored)
glog.V(logger.Info).Infof("imported %5d receipts in %9v. #%d [%x… / %x…]%s", stats.processed, common.PrettyDuration(time.Since(start)), last.Number(), first.Hash().Bytes()[:4], last.Hash().Bytes()[:4], ignoredStr)
return 0, nil
}
// updateReceiptAddressBlooms integrates the addresses of all logs in the given set of receipts
// into the given database.
func updateReceiptAddressBlooms(chainDb ethdb.Database, receipts map[uint64]types.Receipts) error {
blooms := make(map[string]*boom.BloomFilter)
for blockNum, receipts := range receipts {
for _, receipt := range receipts {
for depth := 0; depth < len(AddrBloomMapLevels); depth++ {
key := BloomLogsKey(depth, blockNum)
f, found := blooms[string(key)]
if !found {
count, ratio := AddrBloomMapCount[depth], AddrBloomMapRatio[depth]
f = boom.NewBloomFilter(count, ratio)
if data, err := chainDb.Get(key); err == nil {
if err := f.GobDecode(data); err != nil {
return err
}
}
blooms[string(key)] = f
}
for _, log := range receipt.Logs {
f.Add(log.Address.Bytes())
}
}
}
}
batch := chainDb.NewBatch()
for key, filter := range blooms {
data, err := filter.GobEncode()
if err != nil {
return err
}
batch.Put([]byte(key), data)
}
return batch.Write()
}
// WriteBlock writes the block to the chain.
func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err error) {
self.wg.Add(1)

204
core/database_util.go
View file

@ -23,6 +23,7 @@ import (
"errors"
"fmt"
"math/big"
"sync"
"github.com/ethereum/go-ethereum/common"
@ -33,6 +34,7 @@ import (
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/tylertreat/BoomFilters"
)
var (
@ -51,8 +53,10 @@ var (
txMetaSuffix = []byte{0x01}
receiptsPrefix = []byte("receipts-")
mipmapPre = []byte("mipmap-log-bloom-")
MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000}
AddrBloomMapPre = []byte("blm-log-addr-")
AddrBloomMapLevels = []uint64{500000, 100000, 25000, 2500, 50}
AddrBloomMapCount = []uint{500000, 250000, 75000, 15000, 2000}
AddrBloomMapRatio = []float64{0.5, 0.5, 0.25, 0.1, 0.01}
configPrefix = []byte("ethereum-config-") // config prefix for the db
@ -66,9 +70,6 @@ var (
oldBlockHashPrefix = []byte("block-hash-") // [deprecated by the header/block split, remove eventually]
ChainConfigNotFoundErr = errors.New("ChainConfig not found") // general config not found error
mipmapBloomMu sync.Mutex // protect against race condition when updating mipmap blooms
preimageCounter = metrics.NewCounter("db/preimage/total")
preimageHitCounter = metrics.NewCounter("db/preimage/hits")
)
@ -353,22 +354,47 @@ func WriteHeader(db ethdb.Database, header *types.Header) error {
return nil
}
// WriteBody serializes the body of a block into the database.
func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.Body) error {
// WriteBodyBatch serializes the body of a block into the batch.
func WriteBodyBatch(batch ethdb.Batch, hash common.Hash, number uint64, body *types.Body) error {
data, err := rlp.EncodeToBytes(body)
if err != nil {
return err
}
return WriteBodyRLP(db, hash, number, data)
return WriteBodyRLPBatch(batch, hash, number, data)
}
// WriteBody serializes the body of a block into the database.
func WriteBody(db ethdb.Database, hash common.Hash, number uint64, body *types.Body) error {
batch := db.NewBatch()
err := WriteBodyBatch(batch, hash, number, body)
if err != nil {
return err
}
if err := batch.Write(); err != nil {
glog.Fatalf("failed to store block body into database: %v", err)
}
return nil
}
// WriteBodyRLPBatch writes a serialized body of a block into the batch.
func WriteBodyRLPBatch(batch ethdb.Batch, hash common.Hash, number uint64, rlp rlp.RawValue) error {
key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
if err := batch.Put(key, rlp); err != nil {
return fmt.Errorf("failed to store block body into batch: %v", err)
}
glog.V(logger.Debug).Infof("stored block body [%x…]", hash.Bytes()[:4])
return nil
}
// WriteBodyRLP writes a serialized body of a block into the database.
func WriteBodyRLP(db ethdb.Database, hash common.Hash, number uint64, rlp rlp.RawValue) error {
key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
if err := db.Put(key, rlp); err != nil {
batch := db.NewBatch()
if err := WriteBodyRLPBatch(batch, hash, number, rlp); err != nil {
return err
}
if err := batch.Write(); err != nil {
glog.Fatalf("failed to store block body into database: %v", err)
}
glog.V(logger.Debug).Infof("stored block body [%x…]", hash.Bytes()[:4])
return nil
}
@ -399,10 +425,10 @@ func WriteBlock(db ethdb.Database, block *types.Block) error {
return nil
}
// WriteBlockReceipts stores all the transaction receipts belonging to a block
// WriteBlockReceiptsBatch stores all the transaction receipts belonging to a block
// as a single receipt slice. This is used during chain reorganisations for
// rescheduling dropped transactions.
func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, receipts types.Receipts) error {
func WriteBlockReceiptsBatch(batch ethdb.Batch, hash common.Hash, number uint64, receipts types.Receipts) error {
// Convert the receipts into their storage form and serialize them
storageReceipts := make([]*types.ReceiptForStorage, len(receipts))
for i, receipt := range receipts {
@ -414,20 +440,30 @@ func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, rece
}
// Store the flattened receipt slice
key := append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
if err := db.Put(key, bytes); err != nil {
glog.Fatalf("failed to store block receipts into database: %v", err)
if err := batch.Put(key, bytes); err != nil {
return err
}
glog.V(logger.Debug).Infof("stored block receipts [%x…]", hash.Bytes()[:4])
return nil
}
// WriteTransactions stores the transactions associated with a specific block
// into the given database. Beside writing the transaction, the function also
// stores a metadata entry along with the transaction, detailing the position
// of this within the blockchain.
func WriteTransactions(db ethdb.Database, block *types.Block) error {
// WriteBlockReceipts stores all the transaction receipts belonging to a block
// as a single receipt slice. This is used during chain reorganisations for
// rescheduling dropped transactions.
func WriteBlockReceipts(db ethdb.Database, hash common.Hash, number uint64, receipts types.Receipts) error {
batch := db.NewBatch()
err := WriteBlockReceiptsBatch(batch, hash, number, receipts)
if err != nil {
glog.Fatalf("failed to store block receipts into database: %v", err)
}
return batch.Write()
}
// WriteTransactionsBatch stores the transactions associated with a specific
// block into the given batch. Beside writing the transaction, the function
// also stores a metadata entry along with the transaction, detailing the
// position of this within the blockchain.
func WriteTransactionsBatch(batch ethdb.Batch, block *types.Block) error {
// Iterate over each transaction and encode it with its metadata
for i, tx := range block.Transactions() {
// Encode and queue up the transaction for storage
@ -456,10 +492,24 @@ func WriteTransactions(db ethdb.Database, block *types.Block) error {
return err
}
}
return nil
}
// WriteTransactions stores the transactions associated with a specific block
// into the given database. Beside writing the transaction, the function also
// stores a metadata entry along with the transaction, detailing the position
// of this within the blockchain.
func WriteTransactions(db ethdb.Database, block *types.Block) error {
batch := db.NewBatch()
if err := WriteTransactionsBatch(batch, block); err != nil {
return err
}
// Write the scheduled data into the database
if err := batch.Write(); err != nil {
glog.Fatalf("failed to store transactions into database: %v", err)
}
return nil
}
@ -473,10 +523,8 @@ func WriteReceipt(db ethdb.Database, receipt *types.Receipt) error {
return db.Put(append(receiptsPrefix, receipt.TxHash.Bytes()...), data)
}
// WriteReceipts stores a batch of transaction receipts into the database.
func WriteReceipts(db ethdb.Database, receipts types.Receipts) error {
batch := db.NewBatch()
// WriteReceipts stores a batch of transaction receipts into the db batch.
func WriteReceiptsBatch(batch ethdb.Batch, receipts types.Receipts) error {
// Iterate over all the receipts and queue them for database injection
for _, receipt := range receipts {
storageReceipt := (*types.ReceiptForStorage)(receipt)
@ -488,6 +536,17 @@ func WriteReceipts(db ethdb.Database, receipts types.Receipts) error {
return err
}
}
return nil
}
// WriteReceipts stores a batch of transaction receipts into the database.
func WriteReceipts(db ethdb.Database, receipts types.Receipts) error {
batch := db.NewBatch()
err := WriteReceiptsBatch(batch, receipts)
if err != nil {
return err
}
// Write the scheduled data into the database
if err := batch.Write(); err != nil {
glog.Fatalf("failed to store receipts into database: %v", err)
@ -558,48 +617,87 @@ func GetBlockByHashOld(db ethdb.Database, hash common.Hash) *types.Block {
return (*types.Block)(&block)
}
// returns a formatted MIP mapped key by adding prefix, canonical number and level
//
// ex. fn(98, 1000) = (prefix || 1000 || 0)
func mipmapKey(num, level uint64) []byte {
lkey := make([]byte, 8)
binary.BigEndian.PutUint64(lkey, level)
key := new(big.Int).SetUint64(num / level * level)
// BloomLogsKey returns the key that is used to store the address bloom filter
// on the given depth and block height.
func BloomLogsKey(depth int, blockNum uint64) []byte {
level := blockNum / AddrBloomMapLevels[depth] * AddrBloomMapLevels[depth]
return append(mipmapPre, append(lkey, key.Bytes()...)...)
lkey := make([]byte, 10)
binary.BigEndian.PutUint16(lkey[:2], uint16(depth))
binary.BigEndian.PutUint64(lkey[2:], level)
return append(AddrBloomMapPre, lkey...)
}
var (
addMipMapCacheMu sync.RWMutex
addMipMapCache = make(map[string]*boom.BloomFilter)
)
func GetBloomLogs(db ethdb.Database, blockNum uint64, depth int) ([]byte, *boom.BloomFilter, error) {
key := BloomLogsKey(depth, blockNum)
addMipMapCacheMu.Lock()
defer addMipMapCacheMu.Unlock()
f, found := addMipMapCache[string(key)]
if !found {
count, ratio := AddrBloomMapCount[depth], AddrBloomMapRatio[depth]
f = boom.NewBloomFilter(count, ratio)
if data, err := db.Get(key); err == nil {
if err := f.GobDecode(data); err != nil {
return nil, nil, err
}
}
addMipMapCache[string(key)] = f
}
return key, f, nil
}
// WriteMipmapBloomBatch writes each address included in the receipts' logs to the batch.
func WriteMipmapBloomBatch(db ethdb.Database, batch ethdb.Batch, number uint64, receipts types.Receipts) error {
for depth, _ := range AddrBloomMapLevels {
key, bloom, err := GetBloomLogs(db, number, depth)
if err != nil {
return err
}
for _, receipt := range receipts {
for _, log := range receipt.Logs {
bloom.Add(log.Address.Bytes())
}
}
data, err := bloom.GobEncode()
if err != nil {
return err
}
batch.Put(key, data)
addMipMapCacheMu.Lock()
addMipMapCache[string(key)] = bloom
addMipMapCacheMu.Unlock()
}
return nil
}
// WriteMapmapBloom writes each address included in the receipts' logs to the
// MIP bloom bin.
func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts) error {
mipmapBloomMu.Lock()
defer mipmapBloomMu.Unlock()
batch := db.NewBatch()
for _, level := range MIPMapLevels {
key := mipmapKey(number, level)
bloomDat, _ := db.Get(key)
bloom := types.BytesToBloom(bloomDat)
for _, receipt := range receipts {
for _, log := range receipt.Logs {
bloom.Add(log.Address.Big())
}
}
batch.Put(key, bloom.Bytes())
err := WriteMipmapBloomBatch(db, batch, number, receipts)
if err != nil {
return err
}
if err := batch.Write(); err != nil {
return fmt.Errorf("mipmap write fail for: %d: %v", number, err)
glog.Fatalf("mipmap write fail for: %d: %v", number, err)
}
return nil
}
// GetMipmapBloom returns a bloom filter using the number and level as input
// parameters. For available levels see MIPMapLevels.
func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom {
bloomDat, _ := db.Get(mipmapKey(number, level))
return types.BytesToBloom(bloomDat)
}
// PreimageTable returns a Database instance with the key prefix for preimage entries.
func PreimageTable(db ethdb.Database) ethdb.Database {
return ethdb.NewTable(db, preimagePrefix)

25
core/database_util_test.go
View file

@ -517,10 +517,12 @@ func TestMipmapBloom(t *testing.T) {
WriteMipmapBloom(db, 1, types.Receipts{receipt1})
WriteMipmapBloom(db, 2, types.Receipts{receipt2})
for _, level := range MIPMapLevels {
bloom := GetMipmapBloom(db, 2, level)
if !bloom.Test(new(big.Int).SetBytes([]byte("address1"))) {
t.Error("expected test to be included on level:", level)
for depth, _ := range AddrBloomMapLevels {
_, bloom, err := GetBloomLogs(db, 2, depth)
if err != nil {
t.Errorf("could not load bloom: %v", err)
} else if !bloom.Test(common.BytesToAddress([]byte("address")).Bytes()) {
t.Error("expected test to be included on depth:", depth)
}
}
@ -538,8 +540,12 @@ func TestMipmapBloom(t *testing.T) {
}
WriteMipmapBloom(db, 1000, types.Receipts{receipt})
bloom := GetMipmapBloom(db, 1000, 1000)
if bloom.TestBytes([]byte("test")) {
_, bloom, err := GetBloomLogs(db, 1000, 3)
if err != nil {
t.Fatalf("could not load bloom: %v", err)
}
if bloom.Test([]byte("test")) {
t.Error("test should not have been included")
}
}
@ -598,8 +604,11 @@ func TestMipmapChain(t *testing.T) {
}
}
bloom := GetMipmapBloom(db, 0, 1000)
if bloom.TestBytes(addr2[:]) {
_, bloom, err := GetBloomLogs(db, AddrBloomMapLevels[0]+1, 0)
if err != nil {
t.Fatalf("could not load bloom: %v", err)
}
if bloom.Test(addr2.Bytes()) {
t.Error("address was included in bloom and should not have")
}
}

3
eth/backend.go
View file

@ -180,9 +180,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
if err := upgradeChainDatabase(chainDb); err != nil {
return nil, err
}
if !config.LightMode {
if err := addMipmapBloomBins(chainDb); err != nil {
return nil, err
}
}
glog.V(logger.Info).Infof("Protocol Versions: %v, Network Id: %v", ProtocolVersions, config.NetworkId)

10
eth/backend_test.go
View file

@ -71,9 +71,13 @@ func TestMipmapUpgrade(t *testing.T) {
t.Fatal(err)
}
bloom := core.GetMipmapBloom(db, 1, core.MIPMapLevels[0])
if (bloom == types.Bloom{}) {
t.Error("got empty bloom filter")
_, bloom, err := core.GetBloomLogs(db, 1, 0)
if err != nil {
t.Fatalf("could not load bloom: %v", err)
}
if !bloom.Test(addr.Bytes()) {
t.Errorf("expected address to be in bloom")
}
data, _ := db.Get([]byte("setting-mipmap-version"))

93
eth/db_upgrade.go
View file

@ -20,8 +20,9 @@ package eth
import (
"bytes"
"encoding/binary"
"fmt"
"math/big"
"runtime"
"sync"
"time"
"github.com/ethereum/go-ethereum/common"
@ -31,6 +32,7 @@ import (
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
"github.com/ethereum/go-ethereum/rlp"
"github.com/tylertreat/BoomFilters"
)
var useSequentialKeys = []byte("dbUpgrade_20160530sequentialKeys")
@ -313,7 +315,7 @@ func upgradeChainDatabase(db ethdb.Database) error {
}
func addMipmapBloomBins(db ethdb.Database) (err error) {
const mipmapVersion uint = 2
const mipmapVersion uint = 3
// check if the version is set. We ignore data for now since there's
// only one version so we can easily ignore it for now
@ -336,21 +338,96 @@ func addMipmapBloomBins(db ethdb.Database) (err error) {
return
}
}()
latestHash := core.GetHeadBlockHash(db)
latestBlock := core.GetBlock(db, latestHash, core.GetBlockNumber(db, latestHash))
if latestBlock == nil { // clean database
return
}
type task struct {
Start uint64
End uint64
}
var (
step = core.AddrBloomMapLevels[0]
processTill = latestBlock.NumberU64() + 1
tasks = make(chan task, 1+ (latestBlock.NumberU64()/step))
wg = sync.WaitGroup{}
)
for i := uint64(0); i <= processTill; i += step {
if i+step <= processTill {
tasks <- task{i, i + step}
} else {
tasks <- task{i, processTill}
}
}
close(tasks)
process := func() {
for t := range tasks {
type LevelFilter struct {
start, end uint64
filter *boom.BloomFilter
}
filters := make([][]LevelFilter, len(core.AddrBloomMapLevels))
// 1. create filters
for i, level := range core.AddrBloomMapLevels {
for n := uint64(0); n < (core.AddrBloomMapLevels[0] / level); n++ {
count, ratio := core.AddrBloomMapCount[i], core.AddrBloomMapRatio[i]
fil := boom.NewBloomFilter(count, ratio)
filters[i] = append(filters[i], LevelFilter{t.Start + (n * level), t.Start + ((n + 1) * level), fil})
}
}
// 2. add log addresses to bloom filters
for blockNum := t.Start; blockNum < t.End; blockNum++ {
hash := core.GetCanonicalHash(db, blockNum)
if (hash == common.Hash{}) {
glog.Fatalf("chain db corrupted, could not find block %d", blockNum)
}
for depth, lvl := range filters {
receipts := core.GetBlockReceipts(db, hash, blockNum)
f := lvl[(blockNum-t.Start)/core.AddrBloomMapLevels[depth]].filter
for _, receipt := range receipts {
for _, log := range receipt.Logs {
f.Add(log.Address.Bytes())
}
}
}
}
// 3. update in database
batch := db.NewBatch()
for depth, lvl := range filters {
for _, filterLvl := range lvl {
data, err := filterLvl.filter.GobEncode()
if err != nil {
glog.Fatalf("Could not serialize bloom filter")
}
batch.Put(core.BloomLogsKey(depth, filterLvl.start), data)
}
}
if err := batch.Write(); err != nil {
glog.Fatalf("Could not update database: %v", err)
}
}
wg.Done()
}
tstart := time.Now()
glog.V(logger.Info).Infoln("upgrading db log bloom bins")
for i := uint64(0); i <= latestBlock.NumberU64(); i++ {
hash := core.GetCanonicalHash(db, i)
if (hash == common.Hash{}) {
return fmt.Errorf("chain db corrupted. Could not find block %d.", i)
}
core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash, i))
for i := 0; i < runtime.GOMAXPROCS(0); i++ {
wg.Add(1)
go process()
}
wg.Wait()
glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart))
return nil
}

23
eth/filters/api.go
View file

@ -334,11 +334,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([
crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64())
}
filter := New(api.backend, api.useMipMap)
filter.SetBeginBlock(crit.FromBlock.Int64())
filter.SetEndBlock(crit.ToBlock.Int64())
filter.SetAddresses(crit.Addresses)
filter.SetTopics(crit.Topics)
filter := New(api.backend, crit, api.useMipMap)
logs, err := filter.Find(ctx)
return returnLogs(logs), err
@ -374,20 +370,15 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty
return nil, fmt.Errorf("filter not found")
}
filter := New(api.backend, api.useMipMap)
if f.crit.FromBlock != nil {
filter.SetBeginBlock(f.crit.FromBlock.Int64())
} else {
filter.SetBeginBlock(rpc.LatestBlockNumber.Int64())
copy := f.crit
if copy.FromBlock == nil {
copy.FromBlock = big.NewInt(rpc.LatestBlockNumber.Int64())
}
if f.crit.ToBlock != nil {
filter.SetEndBlock(f.crit.ToBlock.Int64())
} else {
filter.SetEndBlock(rpc.LatestBlockNumber.Int64())
if copy.ToBlock == nil {
copy.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64())
}
filter.SetAddresses(f.crit.Addresses)
filter.SetTopics(f.crit.Topics)
filter := New(api.backend, copy, api.useMipMap)
logs, err := filter.Find(ctx)
if err != nil {
return nil, err

293
eth/filters/filter.go
View file

@ -17,9 +17,6 @@
package filters
import (
"math"
"time"
"math/big"
"github.com/ethereum/go-ethereum/common"
@ -28,9 +25,11 @@ import (
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/rpc"
"github.com/tylertreat/BoomFilters"
"golang.org/x/net/context"
)
// Backend defines the interface the filter packages needs to retrieve logs.
type Backend interface {
ChainDb() ethdb.Database
EventMux() *event.TypeMux
@ -38,172 +37,209 @@ type Backend interface {
GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
}
// Filter can be used to retrieve and filter logs.
// Filter can be used to search for particular logs
type Filter struct {
backend Backend
be Backend
crit FilterCriteria
useMipMap bool
created time.Time
db ethdb.Database
begin, end int64
addresses []common.Address
topics [][]common.Hash
}
// New creates a new filter which uses a bloom filter on blocks to figure out whether
// a particular block is interesting or not.
// MipMaps allow past blocks to be searched much more efficiently, but are not available
// to light clients.
func New(backend Backend, useMipMap bool) *Filter {
// filterRange helper structure that keeps track of a bloom filter and the range it covers.
type filterRange struct {
begin uint64
end uint64
bloom boom.Filter
}
// New creates a filter that can be used to search for logs that match the given criteria.
// When `useMipMap` is `true` it uses pre-calculated bloom filters to optimize searching.
// These pre-calculated bloom filters are not available in light mode and require at least
// one address in the criteria.
func New(be Backend, crit FilterCriteria, useMipMap bool) *Filter {
if crit.FromBlock == nil || crit.FromBlock.Int64() < 0 {
crit.FromBlock = big.NewInt(0)
}
return &Filter{
backend: backend,
be: be,
useMipMap: useMipMap,
db: backend.ChainDb(),
crit: crit,
}
}
// SetBeginBlock sets the earliest block for filtering.
// -1 = latest block (i.e., the current block)
// hash = particular hash from-to
func (f *Filter) SetBeginBlock(begin int64) {
f.begin = begin
// Find returns all logs matching the filter criteria.
func (f *Filter) Find(ctx context.Context) ([]*types.Log, error) {
if f.useMipMap && len(f.crit.Addresses) > 0 {
return f.findByAddressIdx(ctx)
}
return f.findFullRangeScan(ctx)
}
// SetEndBlock sets the latest block for filtering.
func (f *Filter) SetEndBlock(end int64) {
f.end = end
// findByAddressIdx returns all logs matching the criteria. It uses an bloom filter based
// index on log addresses for faster searching (not available in light mode).
func (f *Filter) findByAddressIdx(ctx context.Context) ([]*types.Log, error) {
var (
depth = 0
start, end uint64
level = core.AddrBloomMapLevels[depth]
logs []*types.Log
filters = []filterRange{}
)
start = f.crit.FromBlock.Uint64()
if f.crit.ToBlock.Int64() < 0 {
h, err := f.be.HeaderByNumber(ctx, rpc.BlockNumber(f.crit.ToBlock.Int64()))
if err != nil {
return nil, err
}
end = h.Number.Uint64()
} else {
end = f.crit.ToBlock.Uint64()
}
// SetAddresses matches only logs that are generated from addresses that are included
// in the given addresses.
func (f *Filter) SetAddresses(addr []common.Address) {
f.addresses = addr
// find top level block ranges that (probably) contains logs from one of the target addresses.
for num := (start / level) * level; num <= end; num += level {
_, bloom, err := core.GetBloomLogs(f.be.ChainDb(), num, depth)
if err != nil {
return logs, nil
}
if f.matchAddress(bloom) {
filters = append(filters, filterRange{num, num + core.AddrBloomMapLevels[depth], bloom})
}
}
// SetTopics matches only logs that have topics matching the given topics.
func (f *Filter) SetTopics(topics [][]common.Hash) {
f.topics = topics
return f.filterRange(ctx, depth+1, filters, start, end)
}
// FindOnce searches the blockchain for matching log entries, returning
// all matching entries from the first block that contains matches,
// updating the start point of the filter accordingly. If no results are
// found, a nil slice is returned.
func (f *Filter) FindOnce(ctx context.Context) ([]*types.Log, error) {
head, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber)
if head == nil {
return nil, nil
}
headBlockNumber := head.Number.Uint64()
var beginBlockNo uint64 = uint64(f.begin)
if f.begin == -1 {
beginBlockNo = headBlockNumber
}
var endBlockNo uint64 = uint64(f.end)
if f.end == -1 {
endBlockNo = headBlockNumber
// filterRange returns the set of logs that match the filter criteria in bf within the block range
// [start, end]. This is a depth first search to guarantee logs are sorted by block number.
func (f *Filter) filterRange(ctx context.Context, depth int, filters []filterRange, start, end uint64) ([]*types.Log, error) {
var logs []*types.Log
for _, filter := range filters {
// reached lowest level, do a full block scan on the filter range
if depth == len(core.AddrBloomMapLevels) {
return f.scanRangeWithFilter(ctx, filter, start, end)
}
// if no addresses are present we can't make use of fast search which
// uses the mipmap bloom filters to check for fast inclusion and uses
// higher range probability in order to ensure at least a false positive
if !f.useMipMap || len(f.addresses) == 0 {
logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo)
f.begin = int64(blockNumber + 1)
return logs, err
// narrow search by evicting branches that are guaranteed not to include logs.
level := core.AddrBloomMapLevels[depth]
for num := (filter.begin / level) * level; num <= end && num < filter.end; num += level {
if num+level < start {
continue
}
_, bloom, err := core.GetBloomLogs(f.be.ChainDb(), num, depth)
if err != nil {
return nil, err
}
logs, blockNumber := f.mipFind(beginBlockNo, endBlockNo, 0)
f.begin = int64(blockNumber + 1)
if f.matchAddress(bloom) {
fr := filterRange{num, num + level, bloom}
l, err := f.filterRange(ctx, depth+1, []filterRange{fr}, start, end)
if err != nil {
return nil, err
}
logs = append(logs, l...)
}
}
}
return logs, nil
}
// Run filters logs with the current parameters set
func (f *Filter) Find(ctx context.Context) (logs []*types.Log, err error) {
for {
newLogs, err := f.FindOnce(ctx)
if len(newLogs) == 0 || err != nil {
// logs is a helper that returns all logs in the specified block that matches the filter criteria.
func (f *Filter) logs(ctx context.Context, blockNum uint64) ([]*types.Log, error) {
var logs []*types.Log
blockNumber := rpc.BlockNumber(blockNum)
header, err := f.be.HeaderByNumber(ctx, blockNumber)
if header == nil || err != nil {
return logs, err
}
logs = append(logs, newLogs...)
}
}
func (f *Filter) mipFind(start, end uint64, depth int) (logs []*types.Log, blockNumber uint64) {
level := core.MIPMapLevels[depth]
// normalise numerator so we can work in level specific batches and
// work with the proper range checks
for num := start / level * level; num <= end; num += level {
// find addresses in bloom filters
bloom := core.GetMipmapBloom(f.db, num, level)
// Don't bother checking the first time through the loop - we're probably picking
// up where a previous run left off.
first := true
for _, addr := range f.addresses {
if first || bloom.TestBytes(addr[:]) {
first = false
// range check normalised values and make sure that
// we're resolving the correct range instead of the
// normalised values.
start := uint64(math.Max(float64(num), float64(start)))
end := uint64(math.Min(float64(num+level-1), float64(end)))
if depth+1 == len(core.MIPMapLevels) {
l, blockNumber, _ := f.getLogs(context.Background(), start, end)
if len(l) > 0 {
return l, blockNumber
}
} else {
l, blockNumber := f.mipFind(start, end, depth+1)
if len(l) > 0 {
return l, blockNumber
}
}
}
}
}
return nil, end
}
func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types.Log, blockNumber uint64, err error) {
for i := start; i <= end; i++ {
blockNumber := rpc.BlockNumber(i)
header, err := f.backend.HeaderByNumber(ctx, blockNumber)
if header == nil || err != nil {
return logs, end, err
}
// Use bloom filtering to see if this block is interesting given the
// current parameters
if f.bloomFilter(header.Bloom) {
// Get the logs of the block
receipts, err := f.backend.GetReceipts(ctx, header.Hash())
if BloomFilter(header.Bloom, f.crit.Addresses, f.crit.Topics) {
receipts, err := f.be.GetReceipts(ctx, header.Hash())
if err != nil {
return nil, end, err
return nil, err
}
var unfiltered []*types.Log
for _, receipt := range receipts {
unfiltered = append(unfiltered, ([]*types.Log)(receipt.Logs)...)
}
logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics)
if len(logs) > 0 {
return logs, uint64(blockNumber), nil
logs = append(logs, filterLogs(unfiltered, nil, nil, f.crit.Addresses, f.crit.Topics)...)
}
return logs, nil
}
// scanRangeWithFilter returns all logs matching the bf filter criteria.
func (f *Filter) scanRangeWithFilter(ctx context.Context, filter filterRange, start, end uint64) ([]*types.Log, error) {
if start < filter.begin {
start = filter.begin
}
// in case the toBlock is within the filter range make it inclusive.
if end > filter.begin && end <= filter.end {
end += 1
} else if end > filter.end {
end = filter.end
}
var logs []*types.Log
for i := start; i < end; i++ {
if l, err := f.logs(ctx, i); err == nil {
logs = append(logs, l...)
} else {
return nil, err
}
}
return logs, end, nil
return logs, nil
}
// findFullRangeScan performance a naive full blocks scan in the range [start, end] and
// returns logs matching the filters criteria.
func (f *Filter) findFullRangeScan(ctx context.Context) ([]*types.Log, error) {
var (
start = f.crit.FromBlock.Uint64()
end uint64
logs []*types.Log
)
if f.crit.ToBlock.Int64() < 0 {
h, err := f.be.HeaderByNumber(ctx, rpc.BlockNumber(f.crit.ToBlock.Int64()))
if err != nil {
return nil, err
}
end = h.Number.Uint64()
} else {
end = f.crit.ToBlock.Uint64()
}
for i := start; i < end; i++ {
if l, err := f.logs(ctx, i); err == nil {
logs = append(logs, l...)
} else {
return nil, err
}
}
return logs, nil
}
// matchAddress returns an indication if at least one of the addresses
// in bf.addresses is stored in the given bloom.
func (f *Filter) matchAddress(bloom boom.Filter) bool {
for _, addr := range f.crit.Addresses {
if bloom.Test(addr.Bytes()) {
return true
}
}
return false
}
// includes returns an indication if a is included in addresses.
func includes(addresses []common.Address, a common.Address) bool {
for _, addr := range addresses {
if addr == a {
return true
}
}
return false
}
@ -251,11 +287,8 @@ Logs:
return ret
}
func (f *Filter) bloomFilter(bloom types.Bloom) bool {
return bloomFilter(bloom, f.addresses, f.topics)
}
func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]common.Hash) bool {
// BloomFilter returns an indication if the given addresses and topics match with the given bloom
func BloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]common.Hash) bool {
if len(addresses) > 0 {
var included bool
for _, addr := range addresses {
@ -264,7 +297,6 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo
break
}
}
if !included {
return false
}
@ -282,6 +314,5 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo
return false
}
}
return true
}

2
eth/filters/filter_system.go
View file

@ -370,7 +370,7 @@ func (es *EventSystem) lightFilterNewHead(newHeader *types.Header, callBack func
// filter logs of a single header in light client mode
func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common.Address, topics [][]common.Hash, remove bool) []*types.Log {
if bloomFilter(header.Bloom, addresses, topics) {
if BloomFilter(header.Bloom, addresses, topics) {
// Get the logs of the block
ctx, _ := context.WithTimeout(context.Background(), time.Second*5)
receipts, err := es.backend.GetReceipts(ctx, header.Hash())

60
eth/filters/filter_test.go
View file

@ -105,11 +105,13 @@ func BenchmarkMipmaps(b *testing.B) {
}
b.ResetTimer()
filter := New(backend, true)
filter.SetAddresses([]common.Address{addr1, addr2, addr3, addr4})
filter.SetBeginBlock(0)
filter.SetEndBlock(-1)
crit := FilterCriteria{
FromBlock: big.NewInt(0),
ToBlock: big.NewInt(-1),
Addresses: []common.Address{addr1, addr2, addr3, addr4},
}
filter := New(backend, crit, true)
for i := 0; i < b.N; i++ {
logs, _ := filter.Find(context.Background())
if len(logs) != 4 {
@ -208,22 +210,22 @@ func TestFilters(t *testing.T) {
}
}
filter := New(backend, true)
filter.SetAddresses([]common.Address{addr})
filter.SetTopics([][]common.Hash{{hash1, hash2, hash3, hash4}})
filter.SetBeginBlock(0)
filter.SetEndBlock(-1)
toCrit := func(addresses []common.Address, topics [][]common.Hash, begin, end int64) FilterCriteria {
return FilterCriteria{
FromBlock: big.NewInt(begin),
ToBlock: big.NewInt(end),
Addresses: addresses,
Topics: topics,
}
}
filter := New(backend, toCrit([]common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}, 0, -1), true)
logs, _ := filter.Find(context.Background())
if len(logs) != 4 {
t.Error("expected 4 log, got", len(logs))
}
filter = New(backend, true)
filter.SetAddresses([]common.Address{addr})
filter.SetTopics([][]common.Hash{{hash3}})
filter.SetBeginBlock(900)
filter.SetEndBlock(999)
filter = New(backend, toCrit([]common.Address{addr}, [][]common.Hash{{hash3}}, 900, 999), true)
logs, _ = filter.Find(context.Background())
if len(logs) != 1 {
t.Error("expected 1 log, got", len(logs))
@ -232,11 +234,7 @@ func TestFilters(t *testing.T) {
t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0])
}
filter = New(backend, true)
filter.SetAddresses([]common.Address{addr})
filter.SetTopics([][]common.Hash{{hash3}})
filter.SetBeginBlock(990)
filter.SetEndBlock(-1)
filter = New(backend, toCrit([]common.Address{addr}, [][]common.Hash{{hash3}}, 990, -1), true)
logs, _ = filter.Find(context.Background())
if len(logs) != 1 {
t.Error("expected 1 log, got", len(logs))
@ -245,43 +243,27 @@ func TestFilters(t *testing.T) {
t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0])
}
filter = New(backend, true)
filter.SetTopics([][]common.Hash{{hash1, hash2}})
filter.SetBeginBlock(1)
filter.SetEndBlock(10)
filter = New(backend, toCrit([]common.Address{}, [][]common.Hash{{hash1, hash2}}, 1, 10), true)
logs, _ = filter.Find(context.Background())
if len(logs) != 2 {
t.Error("expected 2 log, got", len(logs))
}
failHash := common.BytesToHash([]byte("fail"))
filter = New(backend, true)
filter.SetTopics([][]common.Hash{{failHash}})
filter.SetBeginBlock(0)
filter.SetEndBlock(-1)
filter = New(backend, toCrit([]common.Address{}, [][]common.Hash{{failHash}}, 0, -1), true)
logs, _ = filter.Find(context.Background())
if len(logs) != 0 {
t.Error("expected 0 log, got", len(logs))
}
failAddr := common.BytesToAddress([]byte("failmenow"))
filter = New(backend, true)
filter.SetAddresses([]common.Address{failAddr})
filter.SetBeginBlock(0)
filter.SetEndBlock(-1)
filter = New(backend, toCrit([]common.Address{failAddr}, [][]common.Hash{{}}, 0, -1), true)
logs, _ = filter.Find(context.Background())
if len(logs) != 0 {
t.Error("expected 0 log, got", len(logs))
}
filter = New(backend, true)
filter.SetTopics([][]common.Hash{{failHash}, {hash1}})
filter.SetBeginBlock(0)
filter.SetEndBlock(-1)
filter = New(backend, toCrit([]common.Address{}, [][]common.Hash{{failHash}, {hash1}}, 0, -1), true)
logs, _ = filter.Find(context.Background())
if len(logs) != 0 {
t.Error("expected 0 log, got", len(logs))

24
vendor/github.com/tylertreat/BoomFilters/.gitignore generated vendored Normal file
View file

@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof

12
vendor/github.com/tylertreat/BoomFilters/.travis.yml generated vendored Normal file
View file

@ -0,0 +1,12 @@
language: go
go:
- 1.3
- 1.4
- tip
before_install: go get golang.org/x/tools/cmd/cover
script: go test -cover ./...
notifications:
email: false

202
vendor/github.com/tylertreat/BoomFilters/LICENSE generated vendored Normal file
View file

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

429
vendor/github.com/tylertreat/BoomFilters/README.md generated vendored Normal file
View file

@ -0,0 +1,429 @@
# Boom Filters
[![Build Status](https://travis-ci.org/tylertreat/BoomFilters.svg?branch=master)](https://travis-ci.org/tylertreat/BoomFilters) [![GoDoc](https://godoc.org/github.com/tylertreat/BoomFilters?status.png)](https://godoc.org/github.com/tylertreat/BoomFilters)
**Boom Filters** are probabilistic data structures for [processing continuous, unbounded streams](http://www.bravenewgeek.com/stream-processing-and-probabilistic-methods/). This includes **Stable Bloom Filters**, **Scalable Bloom Filters**, **Counting Bloom Filters**, **Inverse Bloom Filters**, **Cuckoo Filters**, several variants of **traditional Bloom filters**, **HyperLogLog**, **Count-Min Sketch**, and **MinHash**.
Classic Bloom filters generally require a priori knowledge of the data set in order to allocate an appropriately sized bit array. This works well for offline processing, but online processing typically involves unbounded data streams. With enough data, a traditional Bloom filter "fills up", after which it has a false-positive probability of 1.
Boom Filters are useful for situations where the size of the data set isn't known ahead of time. For example, a Stable Bloom Filter can be used to deduplicate events from an unbounded event stream with a specified upper bound on false positives and minimal false negatives. Alternatively, an Inverse Bloom Filter is ideal for deduplicating a stream where duplicate events are relatively close together. This results in no false positives and, depending on how close together duplicates are, a small probability of false negatives. Scalable Bloom Filters place a tight upper bound on false positives while avoiding false negatives but require allocating memory proportional to the size of the data set. Counting Bloom Filters and Cuckoo Filters are useful for cases which require adding and removing elements to and from a set.
For large or unbounded data sets, calculating the exact cardinality is impractical. HyperLogLog uses a fraction of the memory while providing an accurate approximation. Similarly, Count-Min Sketch provides an efficient way to estimate event frequency for data streams, while Top-K tracks the top-k most frequent elements.
MinHash is a probabilistic algorithm to approximate the similarity between two sets. This can be used to cluster or compare documents by splitting the corpus into a bag of words.
## Installation
```
$ go get github.com/tylertreat/BoomFilters
```
## Stable Bloom Filter
This is an implementation of Stable Bloom Filters as described by Deng and Rafiei in [Approximately Detecting Duplicates for Streaming Data using Stable Bloom Filters](http://webdocs.cs.ualberta.ca/~drafiei/papers/DupDet06Sigmod.pdf).
A Stable Bloom Filter (SBF) continuously evicts stale information so that it has room for more recent elements. Like traditional Bloom filters, an SBF has a non-zero probability of false positives, which is controlled by several parameters. Unlike the classic Bloom filter, an SBF has a tight upper bound on the rate of false positives while introducing a non-zero rate of false negatives. The false-positive rate of a classic Bloom filter eventually reaches 1, after which all queries result in a false positive. The stable-point property of an SBF means the false-positive rate asymptotically approaches a configurable fixed constant. A classic Bloom filter is actually a special case of SBF where the eviction rate is zero and the cell size is one, so this provides support for them as well (in addition to bitset-based Bloom filters).
Stable Bloom Filters are useful for cases where the size of the data set isn't known a priori and memory is bounded. For example, an SBF can be used to deduplicate events from an unbounded event stream with a specified upper bound on false positives and minimal false negatives.
### Usage
```go
package main
import (
"fmt"
"github.com/tylertreat/BoomFilters"
)
func main() {
sbf := boom.NewDefaultStableBloomFilter(10000, 0.01)
fmt.Println("stable point", sbf.StablePoint())
sbf.Add([]byte(`a`))
if sbf.Test([]byte(`a`)) {
fmt.Println("contains a")
}
if !sbf.TestAndAdd([]byte(`b`)) {
fmt.Println("doesn't contain b")
}
if sbf.Test([]byte(`b`)) {
fmt.Println("now it contains b!")
}
// Restore to initial state.
sbf.Reset()
}
```
## Scalable Bloom Filter
This is an implementation of a Scalable Bloom Filter as described by Almeida, Baquero, Preguica, and Hutchison in [Scalable Bloom Filters](http://gsd.di.uminho.pt/members/cbm/ps/dbloom.pdf).
A Scalable Bloom Filter (SBF) dynamically adapts to the size of the data set while enforcing a tight upper bound on the rate of false positives and a false-negative probability of zero. This works by adding Bloom filters with geometrically decreasing false-positive rates as filters become full. A tightening ratio, r, controls the filter growth. The compounded probability over the whole series converges to a target value, even accounting for an infinite series.
Scalable Bloom Filters are useful for cases where the size of the data set isn't known a priori and memory constraints aren't of particular concern. For situations where memory is bounded, consider using Inverse or Stable Bloom Filters.
The core parts of this implementation were originally written by Jian Zhen as discussed in [Benchmarking Bloom Filters and Hash Functions in Go](http://zhen.org/blog/benchmarking-bloom-filters-and-hash-functions-in-go/).
### Usage
```go
package main
import (
"fmt"
"github.com/tylertreat/BoomFilters"
)
func main() {
sbf := boom.NewDefaultScalableBloomFilter(0.01)
sbf.Add([]byte(`a`))
if sbf.Test([]byte(`a`)) {
fmt.Println("contains a")
}
if !sbf.TestAndAdd([]byte(`b`)) {
fmt.Println("doesn't contain b")
}
if sbf.Test([]byte(`b`)) {
fmt.Println("now it contains b!")
}
// Restore to initial state.
sbf.Reset()
}
```
## Inverse Bloom Filter
An Inverse Bloom Filter, or "the opposite of a Bloom filter", is a concurrent, probabilistic data structure used to test whether an item has been observed or not. This implementation, [originally described and written by Jeff Hodges](http://www.somethingsimilar.com/2012/05/21/the-opposite-of-a-bloom-filter/), replaces the use of MD5 hashing with a non-cryptographic FNV-1 function.
The Inverse Bloom Filter may report a false negative but can never report a false positive. That is, it may report that an item has not been seen when it actually has, but it will never report an item as seen which it hasn't come across. This behaves in a similar manner to a fixed-size hashmap which does not handle conflicts.
This structure is particularly well-suited to streams in which duplicates are relatively close together. It uses a CAS-style approach, which makes it thread-safe.
### Usage
```go
package main
import (
"fmt"
"github.com/tylertreat/BoomFilters"
)
func main() {
ibf := boom.NewInverseBloomFilter(10000)
ibf.Add([]byte(`a`))
if ibf.Test([]byte(`a`)) {
fmt.Println("contains a")
}
if !ibf.TestAndAdd([]byte(`b`)) {
fmt.Println("doesn't contain b")
}
if ibf.Test([]byte(`b`)) {
fmt.Println("now it contains b!")
}
}
```
## Counting Bloom Filter
This is an implementation of a Counting Bloom Filter as described by Fan, Cao, Almeida, and Broder in [Summary Cache: A Scalable Wide-Area Web Cache Sharing Protocol](http://pages.cs.wisc.edu/~jussara/papers/00ton.pdf).
A Counting Bloom Filter (CBF) provides a way to remove elements by using an array of n-bit buckets. When an element is added, the respective buckets are incremented. To remove an element, the respective buckets are decremented. A query checks that each of the respective buckets are non-zero. Because CBFs allow elements to be removed, they introduce a non-zero probability of false negatives in addition to the possibility of false positives.
Counting Bloom Filters are useful for cases where elements are both added and removed from the data set. Since they use n-bit buckets, CBFs use roughly n-times more memory than traditional Bloom filters.
See Deletable Bloom Filter for an alternative which avoids false negatives.
### Usage
```go
package main
import (
"fmt"
"github.com/tylertreat/BoomFilters"
)
func main() {
bf := boom.NewDefaultCountingBloomFilter(1000, 0.01)
bf.Add([]byte(`a`))
if bf.Test([]byte(`a`)) {
fmt.Println("contains a")
}
if !bf.TestAndAdd([]byte(`b`)) {
fmt.Println("doesn't contain b")
}
if bf.TestAndRemove([]byte(`b`)) {
fmt.Println("removed b")
}
// Restore to initial state.
bf.Reset()
}
```
## Cuckoo Filter
This is an implementation of a Cuckoo Filter as described by Andersen, Kaminsky, and Mitzenmacher in [Cuckoo Filter: Practically Better Than Bloom](http://www.pdl.cmu.edu/PDL-FTP/FS/cuckoo-conext2014.pdf). The Cuckoo Filter is similar to the Counting Bloom Filter in that it supports adding and removing elements, but it does so in a way that doesn't significantly degrade space and performance.
It works by using a cuckoo hashing scheme for inserting items. Instead of storing the elements themselves, it stores their fingerprints which also allows for item removal without false negatives (if you don't attempt to remove an item not contained in the filter).
For applications that store many items and target moderately low false-positive rates, cuckoo filters have lower space overhead than space-optimized Bloom filters.
### Usage
```go
package main
import (
"fmt"
"github.com/tylertreat/BoomFilters"
)
func main() {
cf := boom.NewCuckooFilter(1000, 0.01)
cf.Add([]byte(`a`))
if cf.Test([]byte(`a`)) {
fmt.Println("contains a")
}
if contains, _ := cf.TestAndAdd([]byte(`b`)); !contains {
fmt.Println("doesn't contain b")
}
if cf.TestAndRemove([]byte(`b`)) {
fmt.Println("removed b")
}
// Restore to initial state.
cf.Reset()
}
```
## Classic Bloom Filter
A classic Bloom filter is a special case of a Stable Bloom Filter whose eviction rate is zero and cell size is one. We call this special case an Unstable Bloom Filter. Because cells require more memory overhead, this package also provides two bitset-based Bloom filter variations. The first variation is the traditional implementation consisting of a single bit array. The second implementation is a partitioned approach which uniformly distributes the probability of false positives across all elements.
Bloom filters have a limited capacity, depending on the configured size. Once all bits are set, the probability of a false positive is 1. However, traditional Bloom filters cannot return a false negative.
A Bloom filter is ideal for cases where the data set is known a priori because the false-positive rate can be configured by the size and number of hash functions.
### Usage
```go
package main
import (
"fmt"
"github.com/tylertreat/BoomFilters"
)
func main() {
// We could also use boom.NewUnstableBloomFilter or boom.NewPartitionedBloomFilter.
bf := boom.NewBloomFilter(1000, 0.01)
bf.Add([]byte(`a`))
if bf.Test([]byte(`a`)) {
fmt.Println("contains a")
}
if !bf.TestAndAdd([]byte(`b`)) {
fmt.Println("doesn't contain b")
}
if bf.Test([]byte(`b`)) {
fmt.Println("now it contains b!")
}
// Restore to initial state.
bf.Reset()
}
```
## Count-Min Sketch
This is an implementation of a Count-Min Sketch as described by Cormode and Muthukrishnan in [An Improved Data Stream Summary: The Count-Min Sketch and its Applications](http://dimacs.rutgers.edu/~graham/pubs/papers/cm-full.pdf).
A Count-Min Sketch (CMS) is a probabilistic data structure which approximates the frequency of events in a data stream. Unlike a hash map, a CMS uses sub-linear space at the expense of a configurable error factor. Similar to Counting Bloom filters, items are hashed to a series of buckets, which increment a counter. The frequency of an item is estimated by taking the minimum of each of the item's respective counter values.
Count-Min Sketches are useful for counting the frequency of events in massive data sets or unbounded streams online. In these situations, storing the entire data set or allocating counters for every event in memory is impractical. It may be possible for offline processing, but real-time processing requires fast, space-efficient solutions like the CMS. For approximating set cardinality, refer to the HyperLogLog.
### Usage
```go
package main
import (
"fmt"
"github.com/tylertreat/BoomFilters"
)
func main() {
cms := boom.NewCountMinSketch(0.001, 0.99)
cms.Add([]byte(`alice`)).Add([]byte(`bob`)).Add([]byte(`bob`)).Add([]byte(`frank`))
fmt.Println("frequency of alice", cms.Count([]byte(`alice`)))
fmt.Println("frequency of bob", cms.Count([]byte(`bob`)))
fmt.Println("frequency of frank", cms.Count([]byte(`frank`)))
// Serialization example
buf := new(bytes.Buffer)
n, err := cms.WriteDataTo(buf)
if err != nil {
fmt.Println(err, n)
}
// Restore to initial state.
cms.Reset()
newCMS := boom.NewCountMinSketch(0.001, 0.99)
n, err = newCMS.ReadDataFrom(buf)
if err != nil {
fmt.Println(err, n)
}
fmt.Println("frequency of frank", newCMS.Count([]byte(`frank`)))
}
```
## Top-K
Top-K uses a Count-Min Sketch and min-heap to track the top-k most frequent elements in a stream.
### Usage
```go
package main
import (
"fmt"
"github.com/tylertreat/BoomFilters"
)
func main() {
topk := NewTopK(0.001, 0.99, 5)
topk.Add([]byte(`bob`)).Add([]byte(`bob`)).Add([]byte(`bob`))
topk.Add([]byte(`tyler`)).Add([]byte(`tyler`)).Add([]byte(`tyler`)).Add([]byte(`tyler`))
topk.Add([]byte(`fred`))
topk.Add([]byte(`alice`)).Add([]byte(`alice`)).Add([]byte(`alice`)).Add([]byte(`alice`))
topk.Add([]byte(`james`))
topk.Add([]byte(`fred`))
topk.Add([]byte(`sara`)).Add([]byte(`sara`))
topk.Add([]byte(`bill`))
for i, element := range topk.Elements() {
fmt.Println(i, string(element.Data), element.Freq)
}
// Restore to initial state.
topk.Reset()
}
```
## HyperLogLog
This is an implementation of HyperLogLog as described by Flajolet, Fusy, Gandouet, and Meunier in [HyperLogLog: the analysis of a near-optimal cardinality estimation algorithm](http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf).
HyperLogLog is a probabilistic algorithm which approximates the number of distinct elements in a multiset. It works by hashing values and calculating the maximum number of leading zeros in the binary representation of each hash. If the maximum number of leading zeros is n, the estimated number of distinct elements in the set is 2^n. To minimize variance, the multiset is split into a configurable number of registers, the maximum number of leading zeros is calculated in the numbers in each register, and a harmonic mean is used to combine the estimates.
For large or unbounded data sets, calculating the exact cardinality is impractical. HyperLogLog uses a fraction of the memory while providing an accurate approximation.
This implementation was [originally written by Eric Lesh](https://github.com/eclesh/hyperloglog). Some small changes and additions have been made, including a way to construct a HyperLogLog optimized for a particular relative accuracy and adding FNV hashing. For counting element frequency, refer to the Count-Min Sketch.
### Usage
```go
package main
import (
"fmt"
"github.com/tylertreat/BoomFilters"
)
func main() {
hll, err := boom.NewDefaultHyperLogLog(0.1)
if err != nil {
panic(err)
}
hll.Add([]byte(`alice`)).Add([]byte(`bob`)).Add([]byte(`bob`)).Add([]byte(`frank`))
fmt.Println("count", hll.Count())
// Serialization example
buf := new(bytes.Buffer)
_, err := hll.WriteDataTo(buf)
if err != nil {
fmt.Println(err)
}
// Restore to initial state.
hll.Reset()
newHll, err := boom.NewDefaultHyperLogLog(0.1)
if err != nil {
fmt.Println(err)
}
_, err := newHll.ReadDataFrom(buf)
if err != nil {
fmt.Println(err)
}
fmt.Println("count", newHll.Count())
}
```
## MinHash
This is a variation of the technique for estimating similarity between two sets as presented by Broder in [On the resemblance and containment of documents](http://gatekeeper.dec.com/ftp/pub/dec/SRC/publications/broder/positano-final-wpnums.pdf).
MinHash is a probabilistic algorithm which can be used to cluster or compare documents by splitting the corpus into a bag of words. MinHash returns the approximated similarity ratio of the two bags. The similarity is less accurate for very small bags of words.
### Usage
```go
package main
import (
"fmt"
"github.com/tylertreat/BoomFilters"
)
func main() {
bag1 := []string{"bill", "alice", "frank", "bob", "sara", "tyler", "james"}
bag2 := []string{"bill", "alice", "frank", "bob", "sara"}
fmt.Println("similarity", boom.MinHash(bag1, bag2))
}
```
## References
- [Approximately Detecting Duplicates for Streaming Data using Stable Bloom Filters](http://webdocs.cs.ualberta.ca/~drafiei/papers/DupDet06Sigmod.pdf)
- [Scalable Bloom Filters](http://gsd.di.uminho.pt/members/cbm/ps/dbloom.pdf)
- [The Opposite of a Bloom Filter](http://www.somethingsimilar.com/2012/05/21/the-opposite-of-a-bloom-filter/)
- [Benchmarking Bloom Filters and Hash Functions in Go](http://zhen.org/blog/benchmarking-bloom-filters-and-hash-functions-in-go/)
- [Summary Cache: A Scalable Wide-Area Web Cache Sharing Protocol](http://pages.cs.wisc.edu/~jussara/papers/00ton.pdf)
- [An Improved Data Stream Summary: The Count-Min Sketch and its Applications](http://dimacs.rutgers.edu/~graham/pubs/papers/cm-full.pdf)
- [HyperLogLog: the analysis of a near-optimal cardinality estimation algorithm](http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf)
- [Package hyperloglog](https://github.com/eclesh/hyperloglog)
- [On the resemblance and containment of documents](http://gatekeeper.dec.com/ftp/pub/dec/SRC/publications/broder/positano-final-wpnums.pdf)
- [Cuckoo Filter: Practically Better Than Bloom](http://www.pdl.cmu.edu/PDL-FTP/FS/cuckoo-conext2014.pdf)

83
vendor/github.com/tylertreat/BoomFilters/boom.go generated vendored Normal file
View file

@ -0,0 +1,83 @@
/*
Package boom implements probabilistic data structures for processing
continuous, unbounded data streams. This includes Stable Bloom Filters,
Scalable Bloom Filters, Counting Bloom Filters, Inverse Bloom Filters, several
variants of traditional Bloom filters, HyperLogLog, Count-Min Sketch, and
MinHash.
Classic Bloom filters generally require a priori knowledge of the data set
in order to allocate an appropriately sized bit array. This works well for
offline processing, but online processing typically involves unbounded data
streams. With enough data, a traditional Bloom filter "fills up", after
which it has a false-positive probability of 1.
Boom Filters are useful for situations where the size of the data set isn't
known ahead of time. For example, a Stable Bloom Filter can be used to
deduplicate events from an unbounded event stream with a specified upper
bound on false positives and minimal false negatives. Alternatively, an
Inverse Bloom Filter is ideal for deduplicating a stream where duplicate
events are relatively close together. This results in no false positives
and, depending on how close together duplicates are, a small probability of
false negatives. Scalable Bloom Filters place a tight upper bound on false
positives while avoiding false negatives but require allocating memory
proportional to the size of the data set. Counting Bloom Filters and Cuckoo
Filters are useful for cases which require adding and removing elements to and
from a set.
For large or unbounded data sets, calculating the exact cardinality is
impractical. HyperLogLog uses a fraction of the memory while providing an
accurate approximation. Similarly, Count-Min Sketch provides an efficient way
to estimate event frequency for data streams. TopK tracks the top-k most
frequent elements.
MinHash is a probabilistic algorithm to approximate the similarity between two
sets. This can be used to cluster or compare documents by splitting the corpus
into a bag of words.
*/
package boom
import (
"encoding/binary"
"hash"
"math"
)
const fillRatio = 0.5
// Filter is a probabilistic data structure which is used to test the
// membership of an element in a set.
type Filter interface {
// Test will test for membership of the data and returns true if it is a
// member, false if not.
Test([]byte) bool
// Add will add the data to the Bloom filter. It returns the filter to
// allow for chaining.
Add([]byte) Filter
// TestAndAdd is equivalent to calling Test followed by Add. It returns
// true if the data is a member, false if not.
TestAndAdd([]byte) bool
}
// OptimalM calculates the optimal Bloom filter size, m, based on the number of
// items and the desired rate of false positives.
func OptimalM(n uint, fpRate float64) uint {
return uint(math.Ceil(float64(n) / ((math.Log(fillRatio) *
math.Log(1-fillRatio)) / math.Abs(math.Log(fpRate)))))
}
// OptimalK calculates the optimal number of hash functions to use for a Bloom
// filter based on the desired rate of false positives.
func OptimalK(fpRate float64) uint {
return uint(math.Ceil(math.Log2(1 / fpRate)))
}
// hashKernel returns the upper and lower base hash values from which the k
// hashes are derived.
func hashKernel(data []byte, hash hash.Hash64) (uint32, uint32) {
hash.Write(data)
sum := hash.Sum(nil)
hash.Reset()
return binary.BigEndian.Uint32(sum[4:8]), binary.BigEndian.Uint32(sum[0:4])
}

182
vendor/github.com/tylertreat/BoomFilters/buckets.go generated vendored Normal file
View file

@ -0,0 +1,182 @@
package boom
import (
"bytes"
"encoding/binary"
"io"
)
// Buckets is a fast, space-efficient array of buckets where each bucket can
// store up to a configured maximum value.
type Buckets struct {
data []byte
bucketSize uint8
max uint8
count uint
}
// NewBuckets creates a new Buckets with the provided number of buckets where
// each bucket is the specified number of bits.
func NewBuckets(count uint, bucketSize uint8) *Buckets {
return &Buckets{
count: count,
data: make([]byte, (count*uint(bucketSize)+7)/8),
bucketSize: bucketSize,
max: (1 << bucketSize) - 1,
}
}
// MaxBucketValue returns the maximum value that can be stored in a bucket.
func (b *Buckets) MaxBucketValue() uint8 {
return b.max
}
// Count returns the number of buckets.
func (b *Buckets) Count() uint {
return b.count
}
// Increment will increment the value in the specified bucket by the provided
// delta. A bucket can be decremented by providing a negative delta. The value
// is clamped to zero and the maximum bucket value. Returns itself to allow for
// chaining.
func (b *Buckets) Increment(bucket uint, delta int32) *Buckets {
val := int32(b.getBits(bucket*uint(b.bucketSize), uint(b.bucketSize))) + delta
if val > int32(b.max) {
val = int32(b.max)
} else if val < 0 {
val = 0
}
b.setBits(uint32(bucket)*uint32(b.bucketSize), uint32(b.bucketSize), uint32(val))
return b
}
// Set will set the bucket value. The value is clamped to zero and the maximum
// bucket value. Returns itself to allow for chaining.
func (b *Buckets) Set(bucket uint, value uint8) *Buckets {
if value > b.max {
value = b.max
}
b.setBits(uint32(bucket)*uint32(b.bucketSize), uint32(b.bucketSize), uint32(value))
return b
}
// Get returns the value in the specified bucket.
func (b *Buckets) Get(bucket uint) uint32 {
return b.getBits(bucket*uint(b.bucketSize), uint(b.bucketSize))
}
// Reset restores the Buckets to the original state. Returns itself to allow
// for chaining.
func (b *Buckets) Reset() *Buckets {
b.data = make([]byte, (b.count*uint(b.bucketSize)+7)/8)
return b
}
// getBits returns the bits at the specified offset and length.
func (b *Buckets) getBits(offset, length uint) uint32 {
byteIndex := offset / 8
byteOffset := offset % 8
if byteOffset+length > 8 {
rem := 8 - byteOffset
return b.getBits(offset, rem) | (b.getBits(offset+rem, length-rem) << rem)
}
bitMask := uint32((1 << length) - 1)
return (uint32(b.data[byteIndex]) & (bitMask << byteOffset)) >> byteOffset
}
// setBits sets bits at the specified offset and length.
func (b *Buckets) setBits(offset, length, bits uint32) {
byteIndex := offset / 8
byteOffset := offset % 8
if byteOffset+length > 8 {
rem := 8 - byteOffset
b.setBits(offset, rem, bits)
b.setBits(offset+rem, length-rem, bits>>rem)
return
}
bitMask := uint32((1 << length) - 1)
b.data[byteIndex] = byte(uint32(b.data[byteIndex]) & ^(bitMask << byteOffset))
b.data[byteIndex] = byte(uint32(b.data[byteIndex]) | ((bits & bitMask) << byteOffset))
}
// WriteTo writes a binary representation of Buckets to an i/o stream.
// It returns the number of bytes written.
func (b *Buckets) WriteTo(stream io.Writer) (int64, error) {
err := binary.Write(stream, binary.BigEndian, b.bucketSize)
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, b.max)
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, uint64(b.count))
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, uint64(len(b.data)))
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, b.data)
if err != nil {
return 0, err
}
return int64(len(b.data) + 2*binary.Size(uint8(0)) + 2*binary.Size(uint64(0))), err
}
// ReadFrom reads a binary representation of Buckets (such as might
// have been written by WriteTo()) from an i/o stream. It returns the number
// of bytes read.
func (b *Buckets) ReadFrom(stream io.Reader) (int64, error) {
var bucketSize, max uint8
var count, len uint64
err := binary.Read(stream, binary.BigEndian, &bucketSize)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &max)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &count)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &len)
if err != nil {
return 0, err
}
data := make([]byte, len)
err = binary.Read(stream, binary.BigEndian, &data)
if err != nil {
return 0, err
}
b.bucketSize = bucketSize
b.max = max
b.count = uint(count)
b.data = data
return int64(int(len) + 2*binary.Size(uint8(0)) + 2*binary.Size(uint64(0))), nil
}
// GobEncode implements gob.GobEncoder interface.
func (b *Buckets) GobEncode() ([]byte, error) {
var buf bytes.Buffer
_, err := b.WriteTo(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// GobDecode implements gob.GobDecoder interface.
func (b *Buckets) GobDecode(data []byte) error {
buf := bytes.NewBuffer(data)
_, err := b.ReadFrom(buf)
return err
}

199
vendor/github.com/tylertreat/BoomFilters/classic.go generated vendored Normal file
View file

@ -0,0 +1,199 @@
package boom
import (
"bytes"
"encoding/binary"
"hash"
"hash/fnv"
"io"
"math"
)
// BloomFilter implements a classic Bloom filter. A Bloom filter has a non-zero
// probability of false positives and a zero probability of false negatives.
type BloomFilter struct {
buckets *Buckets // filter data
hash hash.Hash64 // hash function (kernel for all k functions)
m uint // filter size
k uint // number of hash functions
count uint // number of items added
}
// NewBloomFilter creates a new Bloom filter optimized to store n items with a
// specified target false-positive rate.
func NewBloomFilter(n uint, fpRate float64) *BloomFilter {
m := OptimalM(n, fpRate)
return &BloomFilter{
buckets: NewBuckets(m, 1),
hash: fnv.New64(),
m: m,
k: OptimalK(fpRate),
}
}
// Capacity returns the Bloom filter capacity, m.
func (b *BloomFilter) Capacity() uint {
return b.m
}
// K returns the number of hash functions.
func (b *BloomFilter) K() uint {
return b.k
}
// Count returns the number of items added to the filter.
func (b *BloomFilter) Count() uint {
return b.count
}
// EstimatedFillRatio returns the current estimated ratio of set bits.
func (b *BloomFilter) EstimatedFillRatio() float64 {
return 1 - math.Exp((-float64(b.count)*float64(b.k))/float64(b.m))
}
// FillRatio returns the ratio of set bits.
func (b *BloomFilter) FillRatio() float64 {
sum := uint32(0)
for i := uint(0); i < b.buckets.Count(); i++ {
sum += b.buckets.Get(i)
}
return float64(sum) / float64(b.m)
}
// Test will test for membership of the data and returns true if it is a
// member, false if not. This is a probabilistic test, meaning there is a
// non-zero probability of false positives but a zero probability of false
// negatives.
func (b *BloomFilter) Test(data []byte) bool {
lower, upper := hashKernel(data, b.hash)
// If any of the K bits are not set, then it's not a member.
for i := uint(0); i < b.k; i++ {
if b.buckets.Get((uint(lower)+uint(upper)*i)%b.m) == 0 {
return false
}
}
return true
}
// Add will add the data to the Bloom filter. It returns the filter to allow
// for chaining.
func (b *BloomFilter) Add(data []byte) Filter {
lower, upper := hashKernel(data, b.hash)
// Set the K bits.
for i := uint(0); i < b.k; i++ {
b.buckets.Set((uint(lower)+uint(upper)*i)%b.m, 1)
}
b.count++
return b
}
// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
// the data is a member, false if not.
func (b *BloomFilter) TestAndAdd(data []byte) bool {
lower, upper := hashKernel(data, b.hash)
member := true
// If any of the K bits are not set, then it's not a member.
for i := uint(0); i < b.k; i++ {
idx := (uint(lower) + uint(upper)*i) % b.m
if b.buckets.Get(idx) == 0 {
member = false
}
b.buckets.Set(idx, 1)
}
b.count++
return member
}
// Reset restores the Bloom filter to its original state. It returns the filter
// to allow for chaining.
func (b *BloomFilter) Reset() *BloomFilter {
b.buckets.Reset()
return b
}
// SetHash sets the hashing function used in the filter.
// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
func (b *BloomFilter) SetHash(h hash.Hash64) {
b.hash = h
}
// WriteTo writes a binary representation of the BloomFilter to an i/o stream.
// It returns the number of bytes written.
func (b *BloomFilter) WriteTo(stream io.Writer) (int64, error) {
err := binary.Write(stream, binary.BigEndian, uint64(b.count))
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, uint64(b.m))
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, uint64(b.k))
if err != nil {
return 0, err
}
writtenSize, err := b.buckets.WriteTo(stream)
if err != nil {
return 0, err
}
return writtenSize + int64(3 * binary.Size(uint64(0))), err
}
// ReadFrom reads a binary representation of BloomFilter (such as might
// have been written by WriteTo()) from an i/o stream. It returns the number
// of bytes read.
func (b *BloomFilter) ReadFrom(stream io.Reader) (int64, error) {
var count, m, k uint64
var buckets Buckets
err := binary.Read(stream, binary.BigEndian, &count)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &m)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &k)
if err != nil {
return 0, err
}
readSize, err := buckets.ReadFrom(stream)
if err != nil {
return 0, err
}
b.count = uint(count)
b.m = uint(m)
b.k = uint(k)
b.buckets = &buckets
return readSize + int64(3 * binary.Size(uint64(0))), nil
}
// GobEncode implements gob.GobEncoder interface.
func (b *BloomFilter) GobEncode() ([]byte, error) {
var buf bytes.Buffer
_, err := b.WriteTo(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// GobDecode implements gob.GobDecoder interface.
func (b *BloomFilter) GobDecode(data []byte) error {
buf := bytes.NewBuffer(data)
_, err := b.ReadFrom(buf)
return err
}

158
vendor/github.com/tylertreat/BoomFilters/counting.go generated vendored Normal file
View file

@ -0,0 +1,158 @@
package boom
import (
"hash"
"hash/fnv"
)
// CountingBloomFilter implements a Counting Bloom Filter as described by Fan,
// Cao, Almeida, and Broder in Summary Cache: A Scalable Wide-Area Web Cache
// Sharing Protocol:
//
// http://pages.cs.wisc.edu/~jussara/papers/00ton.pdf
//
// A Counting Bloom Filter (CBF) provides a way to remove elements by using an
// array of n-bit buckets. When an element is added, the respective buckets are
// incremented. To remove an element, the respective buckets are decremented. A
// query checks that each of the respective buckets are non-zero. Because CBFs
// allow elements to be removed, they introduce a non-zero probability of false
// negatives in addition to the possibility of false positives.
//
// Counting Bloom Filters are useful for cases where elements are both added
// and removed from the data set. Since they use n-bit buckets, CBFs use
// roughly n-times more memory than traditional Bloom filters.
type CountingBloomFilter struct {
buckets *Buckets // filter data
hash hash.Hash64 // hash function (kernel for all k functions)
m uint // number of buckets
k uint // number of hash functions
count uint // number of items in the filter
indexBuffer []uint // buffer used to cache indices
}
// NewCountingBloomFilter creates a new Counting Bloom Filter optimized to
// store n items with a specified target false-positive rate and bucket size.
// If you don't know how many bits to use for buckets, use
// NewDefaultCountingBloomFilter for a sensible default.
func NewCountingBloomFilter(n uint, b uint8, fpRate float64) *CountingBloomFilter {
var (
m = OptimalM(n, fpRate)
k = OptimalK(fpRate)
)
return &CountingBloomFilter{
buckets: NewBuckets(m, b),
hash: fnv.New64(),
m: m,
k: k,
indexBuffer: make([]uint, k),
}
}
// NewDefaultCountingBloomFilter creates a new Counting Bloom Filter optimized
// to store n items with a specified target false-positive rate. Buckets are
// allocated four bits.
func NewDefaultCountingBloomFilter(n uint, fpRate float64) *CountingBloomFilter {
return NewCountingBloomFilter(n, 4, fpRate)
}
// Capacity returns the Bloom filter capacity, m.
func (c *CountingBloomFilter) Capacity() uint {
return c.m
}
// K returns the number of hash functions.
func (c *CountingBloomFilter) K() uint {
return c.k
}
// Count returns the number of items in the filter.
func (c *CountingBloomFilter) Count() uint {
return c.count
}
// Test will test for membership of the data and returns true if it is a
// member, false if not. This is a probabilistic test, meaning there is a
// non-zero probability of false positives and false negatives.
func (c *CountingBloomFilter) Test(data []byte) bool {
lower, upper := hashKernel(data, c.hash)
// If any of the K bits are not set, then it's not a member.
for i := uint(0); i < c.k; i++ {
if c.buckets.Get((uint(lower)+uint(upper)*i)%c.m) == 0 {
return false
}
}
return true
}
// Add will add the data to the Bloom filter. It returns the filter to allow
// for chaining.
func (c *CountingBloomFilter) Add(data []byte) Filter {
lower, upper := hashKernel(data, c.hash)
// Set the K bits.
for i := uint(0); i < c.k; i++ {
c.buckets.Increment((uint(lower)+uint(upper)*i)%c.m, 1)
}
c.count++
return c
}
// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
// the data is a member, false if not.
func (c *CountingBloomFilter) TestAndAdd(data []byte) bool {
lower, upper := hashKernel(data, c.hash)
member := true
// If any of the K bits are not set, then it's not a member.
for i := uint(0); i < c.k; i++ {
idx := (uint(lower) + uint(upper)*i) % c.m
if c.buckets.Get(idx) == 0 {
member = false
}
c.buckets.Increment(idx, 1)
}
c.count++
return member
}
// TestAndRemove will test for membership of the data and remove it from the
// filter if it exists. Returns true if the data was a member, false if not.
func (c *CountingBloomFilter) TestAndRemove(data []byte) bool {
lower, upper := hashKernel(data, c.hash)
member := true
// Set the K bits.
for i := uint(0); i < c.k; i++ {
c.indexBuffer[i] = (uint(lower) + uint(upper)*i) % c.m
if c.buckets.Get(c.indexBuffer[i]) == 0 {
member = false
}
}
if member {
for _, idx := range c.indexBuffer {
c.buckets.Increment(idx, -1)
}
c.count--
}
return member
}
// Reset restores the Bloom filter to its original state. It returns the filter
// to allow for chaining.
func (c *CountingBloomFilter) Reset() *CountingBloomFilter {
c.buckets.Reset()
c.count = 0
return c
}
// SetHash sets the hashing function used in the filter.
// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
func (c *CountingBloomFilter) SetHash(h hash.Hash64) {
c.hash = h
}

217
vendor/github.com/tylertreat/BoomFilters/countmin.go generated vendored Normal file
View file

@ -0,0 +1,217 @@
package boom
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"hash"
"hash/fnv"
"io"
"math"
)
// CountMinSketch implements a Count-Min Sketch as described by Cormode and
// Muthukrishnan in An Improved Data Stream Summary: The Count-Min Sketch and
// its Applications:
//
// http://dimacs.rutgers.edu/~graham/pubs/papers/cm-full.pdf
//
// A Count-Min Sketch (CMS) is a probabilistic data structure which
// approximates the frequency of events in a data stream. Unlike a hash map, a
// CMS uses sub-linear space at the expense of a configurable error factor.
// Similar to Counting Bloom filters, items are hashed to a series of buckets,
// which increment a counter. The frequency of an item is estimated by taking
// the minimum of each of the item's respective counter values.
//
// Count-Min Sketches are useful for counting the frequency of events in
// massive data sets or unbounded streams online. In these situations, storing
// the entire data set or allocating counters for every event in memory is
// impractical. It may be possible for offline processing, but real-time
// processing requires fast, space-efficient solutions like the CMS. For
// approximating set cardinality, refer to the HyperLogLog.
type CountMinSketch struct {
matrix [][]uint64 // count matrix
width uint // matrix width
depth uint // matrix depth
count uint64 // number of items added
epsilon float64 // relative-accuracy factor
delta float64 // relative-accuracy probability
hash hash.Hash64 // hash function (kernel for all depth functions)
}
// NewCountMinSketch creates a new Count-Min Sketch whose relative accuracy is
// within a factor of epsilon with probability delta. Both of these parameters
// affect the space and time complexity.
func NewCountMinSketch(epsilon, delta float64) *CountMinSketch {
var (
width = uint(math.Ceil(math.E / epsilon))
depth = uint(math.Ceil(math.Log(1 / delta)))
matrix = make([][]uint64, depth)
)
for i := uint(0); i < depth; i++ {
matrix[i] = make([]uint64, width)
}
return &CountMinSketch{
matrix: matrix,
width: width,
depth: depth,
epsilon: epsilon,
delta: delta,
hash: fnv.New64(),
}
}
// Epsilon returns the relative-accuracy factor, epsilon.
func (c *CountMinSketch) Epsilon() float64 {
return c.epsilon
}
// Delta returns the relative-accuracy probability, delta.
func (c *CountMinSketch) Delta() float64 {
return c.delta
}
// TotalCount returns the number of items added to the sketch.
func (c *CountMinSketch) TotalCount() uint64 {
return c.count
}
// Add will add the data to the set. Returns the CountMinSketch to allow for
// chaining.
func (c *CountMinSketch) Add(data []byte) *CountMinSketch {
lower, upper := hashKernel(data, c.hash)
// Increment count in each row.
for i := uint(0); i < c.depth; i++ {
c.matrix[i][(uint(lower)+uint(upper)*i)%c.width]++
}
c.count++
return c
}
// Count returns the approximate count for the specified item, correct within
// epsilon * total count with a probability of delta.
func (c *CountMinSketch) Count(data []byte) uint64 {
var (
lower, upper = hashKernel(data, c.hash)
count = uint64(math.MaxUint64)
)
for i := uint(0); i < c.depth; i++ {
count = uint64(math.Min(float64(count),
float64(c.matrix[i][(uint(lower)+uint(upper)*i)%c.width])))
}
return count
}
// Merge combines this CountMinSketch with another. Returns an error if the
// matrix width and depth are not equal.
func (c *CountMinSketch) Merge(other *CountMinSketch) error {
if c.depth != other.depth {
return errors.New("matrix depth must match")
}
if c.width != other.width {
return errors.New("matrix width must match")
}
for i := uint(0); i < c.depth; i++ {
for j := uint(0); j < c.width; j++ {
c.matrix[i][j] += other.matrix[i][j]
}
}
c.count += other.count
return nil
}
// Reset restores the CountMinSketch to its original state. It returns itself
// to allow for chaining.
func (c *CountMinSketch) Reset() *CountMinSketch {
matrix := make([][]uint64, c.depth)
for i := uint(0); i < c.depth; i++ {
matrix[i] = make([]uint64, c.width)
}
c.matrix = matrix
c.count = 0
return c
}
// SetHash sets the hashing function used.
func (c *CountMinSketch) SetHash(h hash.Hash64) {
c.hash = h
}
// WriteDataTo writes a binary representation of the CMS data to
// an io stream. It returns the number of bytes written and error
func (c *CountMinSketch) WriteDataTo(stream io.Writer) (int, error) {
buf := new(bytes.Buffer)
// serialize epsilon and delta as cms configuration check
err := binary.Write(buf, binary.LittleEndian, c.epsilon)
if err != nil {
return 0, err
}
err = binary.Write(buf, binary.LittleEndian, c.delta)
if err != nil {
return 0, err
}
err = binary.Write(buf, binary.LittleEndian, c.count)
if err != nil {
return 0, err
}
// encode matrix
for i := range c.matrix {
err = binary.Write(buf, binary.LittleEndian, c.matrix[i])
if err != nil {
return 0, err
}
}
return stream.Write(buf.Bytes())
}
// ReadDataFrom reads a binary representation of the CMS data written
// by WriteDataTo() from io stream. It returns the number of bytes read
// and error
// If serialized CMS configuration is different it returns error with expected params
func (c *CountMinSketch) ReadDataFrom(stream io.Reader) (int, error) {
var (
count uint64
epsilon, delta float64
)
err := binary.Read(stream, binary.LittleEndian, &epsilon)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.LittleEndian, &delta)
if err != nil {
return 0, err
}
// check if serialized and target cms configurations are same
if c.epsilon != epsilon || c.delta != delta {
return 0, fmt.Errorf("expected cms values for epsilon %f and delta %f", epsilon, delta)
}
err = binary.Read(stream, binary.LittleEndian, &count)
if err != nil {
return 0, err
}
for i := uint(0); i < uint(c.depth); i++ {
err = binary.Read(stream, binary.LittleEndian, c.matrix[i])
}
// count size of matrix and count
size := int(c.depth*c.width)*binary.Size(uint64(0)) + binary.Size(count) + 2*binary.Size(float64(0))
c.count = count
return size, err
}

269
vendor/github.com/tylertreat/BoomFilters/cuckoo.go generated vendored Normal file
View file

@ -0,0 +1,269 @@
package boom
import (
"bytes"
"encoding/binary"
"errors"
"hash"
"hash/fnv"
"math"
"math/rand"
)
// maxNumKicks is the maximum number of relocations to attempt when inserting
// an element before considering the filter full.
const maxNumKicks = 500
// bucket consists of a set of []byte entries.
type bucket [][]byte
// contains indicates if the given fingerprint is contained in one of the
// bucket's entries.
func (b bucket) contains(f []byte) bool {
return b.indexOf(f) != -1
}
// indexOf returns the entry index of the given fingerprint or -1 if it's not
// in the bucket.
func (b bucket) indexOf(f []byte) int {
for i, fingerprint := range b {
if bytes.Equal(f, fingerprint) {
return i
}
}
return -1
}
// getEmptyEntry returns the index of the next available entry in the bucket or
// an error if it's full.
func (b bucket) getEmptyEntry() (int, error) {
for i, fingerprint := range b {
if fingerprint == nil {
return i, nil
}
}
return -1, errors.New("full")
}
// CuckooFilter implements a Cuckoo Bloom filter as described by Andersen,
// Kaminsky, and Mitzenmacher in Cuckoo Filter: Practically Better Than Bloom:
//
// http://www.pdl.cmu.edu/PDL-FTP/FS/cuckoo-conext2014.pdf
//
// A Cuckoo Filter is a Bloom filter variation which provides support for
// removing elements without significantly degrading space and performance. It
// works by using a cuckoo hashing scheme for inserting items. Instead of
// storing the elements themselves, it stores their fingerprints which also
// allows for item removal without false negatives (if you don't attempt to
// remove an item not contained in the filter).
//
// For applications that store many items and target moderately low
// false-positive rates, cuckoo filters have lower space overhead than
// space-optimized Bloom filters.
type CuckooFilter struct {
buckets []bucket
hash hash.Hash32 // hash function (used for fingerprint and hash)
m uint // number of buckets
b uint // number of entries per bucket
f uint // length of fingerprints (in bytes)
count uint // number of items in the filter
n uint // filter capacity
}
// NewCuckooFilter creates a new Cuckoo Bloom filter optimized to store n items
// with a specified target false-positive rate.
func NewCuckooFilter(n uint, fpRate float64) *CuckooFilter {
var (
b = uint(4)
f = calculateF(b, fpRate)
m = power2(n / uint(f) * 8)
buckets = make([]bucket, m)
)
for i := uint(0); i < m; i++ {
buckets[i] = make(bucket, b)
}
return &CuckooFilter{
buckets: buckets,
hash: fnv.New32(),
m: m,
b: b,
f: uint(f),
n: n,
}
}
// Buckets returns the number of buckets.
func (c *CuckooFilter) Buckets() uint {
return c.m
}
// Capacity returns the number of items the filter can store.
func (c *CuckooFilter) Capacity() uint {
return c.n
}
// Count returns the number of items in the filter.
func (c *CuckooFilter) Count() uint {
return c.count
}
// Test will test for membership of the data and returns true if it is a
// member, false if not. This is a probabilistic test, meaning there is a
// non-zero probability of false positives.
func (c *CuckooFilter) Test(data []byte) bool {
i1, i2, f := c.components(data)
// If either bucket contains f, it's a member.
return c.buckets[i1%c.m].contains(f) || c.buckets[i2%c.m].contains(f)
}
// Add will add the data to the Cuckoo Filter. It returns an error if the
// filter is full. If the filter is full, an item is removed to make room for
// the new item. This introduces a possibility for false negatives. To avoid
// this, use Count and Capacity to check if the filter is full before adding an
// item.
func (c *CuckooFilter) Add(data []byte) error {
return c.add(c.components(data))
}
// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
// the data is a member, false if not. An error is returned if the filter is
// full. If the filter is full, an item is removed to make room for the new
// item. This introduces a possibility for false negatives. To avoid this, use
// Count and Capacity to check if the filter is full before adding an item.
func (c *CuckooFilter) TestAndAdd(data []byte) (bool, error) {
i1, i2, f := c.components(data)
// If either bucket contains f, it's a member.
if c.buckets[i1%c.m].contains(f) || c.buckets[i2%c.m].contains(f) {
return true, nil
}
return false, c.add(i1, i2, f)
}
// TestAndRemove will test for membership of the data and remove it from the
// filter if it exists. Returns true if the data was a member, false if not.
func (c *CuckooFilter) TestAndRemove(data []byte) bool {
i1, i2, f := c.components(data)
// Try to remove from bucket[i1].
b1 := c.buckets[i1%c.m]
if idx := b1.indexOf(f); idx != -1 {
b1[idx] = nil
c.count--
return true
}
// Try to remove from bucket[i2].
b2 := c.buckets[i2%c.m]
if idx := b2.indexOf(f); idx != -1 {
b2[idx] = nil
c.count--
return true
}
return false
}
// Reset restores the Bloom filter to its original state. It returns the filter
// to allow for chaining.
func (c *CuckooFilter) Reset() *CuckooFilter {
buckets := make([]bucket, c.m)
for i := uint(0); i < c.m; i++ {
buckets[i] = make(bucket, c.b)
}
c.buckets = buckets
c.count = 0
return c
}
// add will insert the fingerprint into the filter returning an error if the
// filter is full.
func (c *CuckooFilter) add(i1, i2 uint, f []byte) error {
// Try to insert into bucket[i1].
b1 := c.buckets[i1%c.m]
if idx, err := b1.getEmptyEntry(); err == nil {
b1[idx] = f
c.count++
return nil
}
// Try to insert into bucket[i2].
b2 := c.buckets[i2%c.m]
if idx, err := b2.getEmptyEntry(); err == nil {
b2[idx] = f
c.count++
return nil
}
// Must relocate existing items.
i := i1
for n := 0; n < maxNumKicks; n++ {
bucketIdx := i % c.m
entryIdx := rand.Intn(int(c.b))
f, c.buckets[bucketIdx][entryIdx] = c.buckets[bucketIdx][entryIdx], f
i = i ^ uint(binary.BigEndian.Uint32(c.computeHash(f)))
b := c.buckets[i%c.m]
if idx, err := b.getEmptyEntry(); err == nil {
b[idx] = f
c.count++
return nil
}
}
return errors.New("full")
}
// components returns the two hash values used to index into the buckets and
// the fingerprint for the given element.
func (c *CuckooFilter) components(data []byte) (uint, uint, []byte) {
var (
hash = c.computeHash(data)
f = hash[0:c.f]
i1 = uint(binary.BigEndian.Uint32(hash))
i2 = i1 ^ uint(binary.BigEndian.Uint32(c.computeHash(f)))
)
return i1, i2, f
}
// computeHash returns a 32-bit hash value for the given data.
func (c *CuckooFilter) computeHash(data []byte) []byte {
c.hash.Write(data)
hash := c.hash.Sum(nil)
c.hash.Reset()
return hash
}
// SetHash sets the hashing function used in the filter.
// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
func (c *CuckooFilter) SetHash(h hash.Hash32) {
c.hash = h
}
// calculateF returns the optimal fingerprint length in bytes for the given
// bucket size and false-positive rate epsilon.
func calculateF(b uint, epsilon float64) uint {
f := uint(math.Ceil(math.Log(2 * float64(b) / epsilon)))
f = f / 8
if f <= 0 {
f = 1
}
return f
}
// power2 calculates the next power of two for the given value.
func power2(x uint) uint {
x--
x |= x >> 1
x |= x >> 2
x |= x >> 4
x |= x >> 8
x |= x >> 16
x |= x >> 32
x++
return x
}

168
vendor/github.com/tylertreat/BoomFilters/deletable.go generated vendored Normal file
View file

@ -0,0 +1,168 @@
package boom
import (
"hash"
"hash/fnv"
)
// DeletableBloomFilter implements a Deletable Bloom Filter as described by
// Rothenberg, Macapuna, Verdi, Magalhaes in The Deletable Bloom filter - A new
// member of the Bloom family:
//
// http://arxiv.org/pdf/1005.0352.pdf
//
// A Deletable Bloom Filter compactly stores information on collisions when
// inserting elements. This information is used to determine if elements are
// deletable. This design enables false-negative-free deletions at a fraction
// of the cost in memory consumption.
//
// Deletable Bloom Filters are useful for cases which require removing elements
// but cannot allow false negatives. This means they can be safely swapped in
// place of traditional Bloom filters.
type DeletableBloomFilter struct {
buckets *Buckets // filter data
collisions *Buckets // filter collision data
hash hash.Hash64 // hash function (kernel for all k functions)
m uint // filter size
regionSize uint // number of bits in a region
k uint // number of hash functions
count uint // number of items added
indexBuffer []uint // buffer used to cache indices
}
// NewDeletableBloomFilter creates a new DeletableBloomFilter optimized to
// store n items with a specified target false-positive rate. The r value
// determines the number of bits to use to store collision information. This
// controls the deletability of an element. Refer to the paper for selecting an
// optimal value.
func NewDeletableBloomFilter(n, r uint, fpRate float64) *DeletableBloomFilter {
var (
m = OptimalM(n, fpRate)
k = OptimalK(fpRate)
)
return &DeletableBloomFilter{
buckets: NewBuckets(m-r, 1),
collisions: NewBuckets(r, 1),
hash: fnv.New64(),
m: m - r,
regionSize: (m - r) / r,
k: k,
indexBuffer: make([]uint, k),
}
}
// Capacity returns the Bloom filter capacity, m.
func (d *DeletableBloomFilter) Capacity() uint {
return d.m
}
// K returns the number of hash functions.
func (d *DeletableBloomFilter) K() uint {
return d.k
}
// Count returns the number of items added to the filter.
func (d *DeletableBloomFilter) Count() uint {
return d.count
}
// Test will test for membership of the data and returns true if it is a
// member, false if not. This is a probabilistic test, meaning there is a
// non-zero probability of false positives but a zero probability of false
// negatives.
func (d *DeletableBloomFilter) Test(data []byte) bool {
lower, upper := hashKernel(data, d.hash)
// If any of the K bits are not set, then it's not a member.
for i := uint(0); i < d.k; i++ {
if d.buckets.Get((uint(lower)+uint(upper)*i)%d.m) == 0 {
return false
}
}
return true
}
// Add will add the data to the Bloom filter. It returns the filter to allow
// for chaining.
func (d *DeletableBloomFilter) Add(data []byte) Filter {
lower, upper := hashKernel(data, d.hash)
// Set the K bits.
for i := uint(0); i < d.k; i++ {
idx := (uint(lower) + uint(upper)*i) % d.m
if d.buckets.Get(idx) != 0 {
// Collision, set corresponding region bit.
d.collisions.Set(idx/d.regionSize, 1)
} else {
d.buckets.Set(idx, 1)
}
}
d.count++
return d
}
// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
// the data is a member, false if not.
func (d *DeletableBloomFilter) TestAndAdd(data []byte) bool {
lower, upper := hashKernel(data, d.hash)
member := true
// If any of the K bits are not set, then it's not a member.
for i := uint(0); i < d.k; i++ {
idx := (uint(lower) + uint(upper)*i) % d.m
if d.buckets.Get(idx) == 0 {
member = false
} else {
// Collision, set corresponding region bit.
d.collisions.Set(idx/d.regionSize, 1)
}
d.buckets.Set(idx, 1)
}
d.count++
return member
}
// TestAndRemove will test for membership of the data and remove it from the
// filter if it exists. Returns true if the data was a member, false if not.
func (d *DeletableBloomFilter) TestAndRemove(data []byte) bool {
lower, upper := hashKernel(data, d.hash)
member := true
// Set the K bits.
for i := uint(0); i < d.k; i++ {
d.indexBuffer[i] = (uint(lower) + uint(upper)*i) % d.m
if d.buckets.Get(d.indexBuffer[i]) == 0 {
member = false
}
}
if member {
for _, idx := range d.indexBuffer {
if d.collisions.Get(idx/d.regionSize) == 0 {
// Clear only bits located in collision-free zones.
d.buckets.Set(idx, 0)
}
}
d.count--
}
return member
}
// Reset restores the Bloom filter to its original state. It returns the filter
// to allow for chaining.
func (d *DeletableBloomFilter) Reset() *DeletableBloomFilter {
d.buckets.Reset()
d.collisions.Reset()
d.count = 0
return d
}
// SetHash sets the hashing function used in the filter.
// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
func (d *DeletableBloomFilter) SetHash(h hash.Hash64) {
d.hash = h
}

253
vendor/github.com/tylertreat/BoomFilters/hyperloglog.go generated vendored Normal file
View file

@ -0,0 +1,253 @@
/*
Original work Copyright 2013 Eric Lesh
Modified work Copyright 2015 Tyler Treat
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
*/
package boom
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"hash"
"hash/fnv"
"io"
"math"
)
var exp32 = math.Pow(2, 32)
// HyperLogLog implements the HyperLogLog cardinality estimation algorithm as
// described by Flajolet, Fusy, Gandouet, and Meunier in HyperLogLog: the
// analysis of a near-optimal cardinality estimation algorithm:
//
// http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf
//
// HyperLogLog is a probabilistic algorithm which approximates the number of
// distinct elements in a multiset. It works by hashing values and calculating
// the maximum number of leading zeros in the binary representation of each
// hash. If the maximum number of leading zeros is n, the estimated number of
// distinct elements in the set is 2^n. To minimize variance, the multiset is
// split into a configurable number of registers, the maximum number of leading
// zeros is calculated in the numbers in each register, and a harmonic mean is
// used to combine the estimates.
//
// For large or unbounded data sets, calculating the exact cardinality is
// impractical. HyperLogLog uses a fraction of the memory while providing an
// accurate approximation. For counting element frequency, refer to the
// Count-Min Sketch.
type HyperLogLog struct {
registers []uint8 // counter registers
m uint // number of registers
b uint32 // number of bits to calculate register
alpha float64 // bias-correction constant
hash hash.Hash32 // hash function
}
// NewHyperLogLog creates a new HyperLogLog with m registers. Returns an error
// if m isn't a power of two.
func NewHyperLogLog(m uint) (*HyperLogLog, error) {
if (m & (m - 1)) != 0 {
return nil, errors.New("m must be a power of two")
}
return &HyperLogLog{
registers: make([]uint8, m),
m: m,
b: uint32(math.Ceil(math.Log2(float64(m)))),
alpha: calculateAlpha(m),
hash: fnv.New32(),
}, nil
}
// NewDefaultHyperLogLog creates a new HyperLogLog optimized for the specified
// standard error. Returns an error if the number of registers can't be
// calculated for the provided accuracy.
func NewDefaultHyperLogLog(e float64) (*HyperLogLog, error) {
m := math.Pow(1.04/e, 2)
return NewHyperLogLog(uint(math.Pow(2, math.Ceil(math.Log2(m)))))
}
// Add will add the data to the set. Returns the HyperLogLog to allow for
// chaining.
func (h *HyperLogLog) Add(data []byte) *HyperLogLog {
var (
hash = h.calculateHash(data)
k = 32 - h.b
r = calculateRho(hash<<h.b, k)
j = hash >> uint(k)
)
if r > h.registers[j] {
h.registers[j] = r
}
return h
}
// Count returns the approximated cardinality of the set.
func (h *HyperLogLog) Count() uint64 {
sum := 0.0
m := float64(h.m)
for _, val := range h.registers {
sum += 1.0 / math.Pow(2.0, float64(val))
}
estimate := h.alpha * m * m / sum
if estimate <= 5.0/2.0*m {
// Small range correction
v := 0
for _, r := range h.registers {
if r == 0 {
v++
}
}
if v > 0 {
estimate = m * math.Log(m/float64(v))
}
} else if estimate > 1.0/30.0*exp32 {
// Large range correction
estimate = -exp32 * math.Log(1-estimate/exp32)
}
return uint64(estimate)
}
// Merge combines this HyperLogLog with another. Returns an error if the number
// of registers in the two HyperLogLogs are not equal.
func (h *HyperLogLog) Merge(other *HyperLogLog) error {
if h.m != other.m {
return errors.New("number of registers must match")
}
for j, r := range other.registers {
if r > h.registers[j] {
h.registers[j] = r
}
}
return nil
}
// Reset restores the HyperLogLog to its original state. It returns itself to
// allow for chaining.
func (h *HyperLogLog) Reset() *HyperLogLog {
h.registers = make([]uint8, h.m)
return h
}
// calculateHash calculates the 32-bit hash value for the provided data.
func (h *HyperLogLog) calculateHash(data []byte) uint32 {
h.hash.Write(data)
sum := h.hash.Sum32()
h.hash.Reset()
return sum
}
// SetHash sets the hashing function used.
func (h *HyperLogLog) SetHash(ha hash.Hash32) {
h.hash = ha
}
// calculateAlpha calculates the bias-correction constant alpha based on the
// number of registers, m.
func calculateAlpha(m uint) (result float64) {
switch m {
case 16:
result = 0.673
case 32:
result = 0.697
case 64:
result = 0.709
default:
result = 0.7213 / (1.0 + 1.079/float64(m))
}
return result
}
// calculateRho calculates the position of the leftmost 1-bit.
func calculateRho(val, max uint32) uint8 {
r := uint32(1)
for val&0x80000000 == 0 && r <= max {
r++
val <<= 1
}
return uint8(r)
}
// WriteDataTo writes a binary representation of the Hll data to
// an io stream. It returns the number of bytes written and error
func (h *HyperLogLog) WriteDataTo(stream io.Writer) (n int, err error) {
buf := new(bytes.Buffer)
// write register number first
err = binary.Write(buf, binary.LittleEndian, uint64(h.m))
if err != nil {
return
}
err = binary.Write(buf, binary.LittleEndian, h.b)
if err != nil {
return
}
err = binary.Write(buf, binary.LittleEndian, h.alpha)
if err != nil {
return
}
err = binary.Write(buf, binary.LittleEndian, h.registers)
if err != nil {
return
}
n, err = stream.Write(buf.Bytes())
return
}
// ReadDataFrom reads a binary representation of the Hll data written
// by WriteDataTo() from io stream. It returns the number of bytes read
// and error.
// If serialized Hll configuration is different it returns error with expected params
func (h *HyperLogLog) ReadDataFrom(stream io.Reader) (int, error) {
var m uint64
// read register number first
err := binary.Read(stream, binary.LittleEndian, &m)
if err != nil {
return 0, err
}
// check if register number is appropriate
// hll register number should be same with serialized hll
if uint64(h.m) != m {
return 0, fmt.Errorf("expected hll register number %d", m)
}
// set other values
err = binary.Read(stream, binary.LittleEndian, &h.b)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.LittleEndian, &h.alpha)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.LittleEndian, h.registers)
if err != nil {
return 0, err
}
// count size of data in registers + m, b, alpha
size := int(h.m)*binary.Size(uint8(0)) + binary.Size(uint64(0)) + binary.Size(uint32(0)) + binary.Size(float64(0))
return size, err
}

238
vendor/github.com/tylertreat/BoomFilters/inverse.go generated vendored Normal file
View file

@ -0,0 +1,238 @@
/*
Original work Copyright (c) 2012 Jeff Hodges. All rights reserved.
Modified work Copyright (c) 2015 Tyler Treat. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Jeff Hodges nor the names of this project's
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
package boom
import (
"bytes"
"encoding/binary"
"encoding/gob"
"hash"
"hash/fnv"
"io"
"sync"
"sync/atomic"
"unsafe"
)
// InverseBloomFilter is a concurrent "inverse" Bloom filter, which is
// effectively the opposite of a classic Bloom filter. This was originally
// described and written by Jeff Hodges:
//
// http://www.somethingsimilar.com/2012/05/21/the-opposite-of-a-bloom-filter/
//
// The InverseBloomFilter may report a false negative but can never report a
// false positive. That is, it may report that an item has not been seen when
// it actually has, but it will never report an item as seen which it hasn't
// come across. This behaves in a similar manner to a fixed-size hashmap which
// does not handle conflicts.
//
// An example use case is deduplicating events while processing a stream of
// data. Ideally, duplicate events are relatively close together.
type InverseBloomFilter struct {
array []*[]byte
hashPool *sync.Pool
capacity uint
}
// NewInverseBloomFilter creates and returns a new InverseBloomFilter with the
// specified capacity.
func NewInverseBloomFilter(capacity uint) *InverseBloomFilter {
return &InverseBloomFilter{
array: make([]*[]byte, capacity),
hashPool: &sync.Pool{New: func() interface{} { return fnv.New32() }},
capacity: capacity,
}
}
// Test will test for membership of the data and returns true if it is a
// member, false if not. This is a probabilistic test, meaning there is a
// non-zero probability of false negatives but a zero probability of false
// positives. That is, it may return false even though the data was added, but
// it will never return true for data that hasn't been added.
func (i *InverseBloomFilter) Test(data []byte) bool {
index := i.index(data)
indexPtr := (*unsafe.Pointer)(unsafe.Pointer(&i.array[index]))
val := (*[]byte)(atomic.LoadPointer(indexPtr))
if val == nil {
return false
}
return bytes.Equal(*val, data)
}
// Add will add the data to the filter. It returns the filter to allow for
// chaining.
func (i *InverseBloomFilter) Add(data []byte) Filter {
index := i.index(data)
i.getAndSet(index, data)
return i
}
// TestAndAdd is equivalent to calling Test followed by Add atomically. It
// returns true if the data is a member, false if not.
func (i *InverseBloomFilter) TestAndAdd(data []byte) bool {
oldID := i.getAndSet(i.index(data), data)
return bytes.Equal(oldID, data)
}
// Capacity returns the filter capacity.
func (i *InverseBloomFilter) Capacity() uint {
return i.capacity
}
// getAndSet returns the data that was in the slice at the given index after
// putting the new data in the slice at that index, atomically.
func (i *InverseBloomFilter) getAndSet(index uint32, data []byte) []byte {
indexPtr := (*unsafe.Pointer)(unsafe.Pointer(&i.array[index]))
keyUnsafe := unsafe.Pointer(&data)
var oldKey []byte
for {
oldKeyUnsafe := atomic.LoadPointer(indexPtr)
if atomic.CompareAndSwapPointer(indexPtr, oldKeyUnsafe, keyUnsafe) {
oldKeyPtr := (*[]byte)(oldKeyUnsafe)
if oldKeyPtr != nil {
oldKey = *oldKeyPtr
}
break
}
}
return oldKey
}
// index returns the array index for the given data.
func (i *InverseBloomFilter) index(data []byte) uint32 {
hash := i.hashPool.Get().(hash.Hash32)
hash.Write(data)
index := hash.Sum32() % uint32(i.capacity)
hash.Reset()
i.hashPool.Put(hash)
return index
}
// SetHashFactory sets the hashing function factory used in the filter.
func (i *InverseBloomFilter) SetHashFactory(h func() hash.Hash32) {
i.hashPool = &sync.Pool{New: func() interface{} { return h() }}
}
// WriteTo writes a binary representation of the InverseBloomFilter to an i/o stream.
// It returns the number of bytes written.
func (i *InverseBloomFilter) WriteTo(stream io.Writer) (int64, error) {
err := binary.Write(stream, binary.BigEndian, uint64(i.capacity))
if err != nil {
return 0, err
}
// Dereference all pointers to []byte
array := make([][]byte, int(i.capacity))
for b := range i.array {
if i.array[b] != nil {
array[b] = *i.array[b]
} else {
array[b] = nil
}
}
// Encode array into a []byte
var buf bytes.Buffer
gob.NewEncoder(&buf).Encode(array)
serialized := buf.Bytes()
// Write the length of encoded slice
err = binary.Write(stream, binary.BigEndian, int64(len(serialized)))
if err != nil {
return 0, err
}
// Write the serialized bytes
written, err := stream.Write(serialized)
if err != nil {
return 0, err
}
return int64(written) + int64(2*binary.Size(uint64(0))), err
}
// ReadFrom reads a binary representation of InverseBloomFilter (such as might
// have been written by WriteTo()) from an i/o stream. It returns the number
// of bytes read.
func (i *InverseBloomFilter) ReadFrom(stream io.Reader) (int64, error) {
var capacity, size uint64
err := binary.Read(stream, binary.BigEndian, &capacity)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &size)
if err != nil {
return 0, err
}
// Read the encoded slice and decode into [][]byte
encoded := make([]byte, size)
stream.Read(encoded)
buf := bytes.NewBuffer(encoded)
dec := gob.NewDecoder(buf)
decoded := make([][]byte, capacity)
dec.Decode(&decoded)
// Create []*[]byte and point to each item in decoded
decodedWithPointers := make([]*[]byte, capacity)
for p := range decodedWithPointers {
if len(decoded[p]) == 0 {
decodedWithPointers[p] = nil
} else {
decodedWithPointers[p] = &decoded[p]
}
}
i.array = decodedWithPointers
i.capacity = uint(capacity)
return int64(len(encoded)) + int64(2*binary.Size(uint64(0))), nil
}
// GobEncode implements gob.GobEncoder interface.
func (i *InverseBloomFilter) GobEncode() ([]byte, error) {
var buf bytes.Buffer
_, err := i.WriteTo(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// GobDecode implements gob.GobDecoder interface.
func (i *InverseBloomFilter) GobDecode(data []byte) error {
buf := bytes.NewBuffer(data)
_, err := i.ReadFrom(buf)
return err
}

104
vendor/github.com/tylertreat/BoomFilters/minhash.go generated vendored Normal file
View file

@ -0,0 +1,104 @@
package boom
import (
"math"
"math/rand"
)
// MinHash is a variation of the technique for estimating similarity between
// two sets as presented by Broder in On the resemblance and containment of
// documents:
//
// http://gatekeeper.dec.com/ftp/pub/dec/SRC/publications/broder/positano-final-wpnums.pdf
//
// This can be used to cluster or compare documents by splitting the corpus
// into a bag of words. MinHash returns the approximated similarity ratio of
// the two bags. The similarity is less accurate for very small bags of words.
func MinHash(bag1, bag2 []string) float32 {
k := len(bag1) + len(bag2)
hashes := make([]int, k)
for i := 0; i < k; i++ {
a := uint(rand.Int())
b := uint(rand.Int())
c := uint(rand.Int())
x := computeHash(a*b*c, a, b, c)
hashes[i] = int(x)
}
bitMap := bitMap(bag1, bag2)
minHashValues := hashBuckets(2, k)
minHash(bag1, 0, minHashValues, bitMap, k, hashes)
minHash(bag2, 1, minHashValues, bitMap, k, hashes)
return similarity(minHashValues, k)
}
func minHash(bag []string, bagIndex int, minHashValues [][]int,
bitArray map[string][]bool, k int, hashes []int) {
index := 0
for element := range bitArray {
for i := 0; i < k; i++ {
if contains(bag, element) {
hindex := hashes[index]
if hindex < minHashValues[bagIndex][index] {
minHashValues[bagIndex][index] = hindex
}
}
}
index++
}
}
func contains(bag []string, element string) bool {
for _, e := range bag {
if e == element {
return true
}
}
return false
}
func bitMap(bag1, bag2 []string) map[string][]bool {
bitArray := map[string][]bool{}
for _, element := range bag1 {
bitArray[element] = []bool{true, false}
}
for _, element := range bag2 {
if _, ok := bitArray[element]; ok {
bitArray[element] = []bool{true, true}
} else if _, ok := bitArray[element]; !ok {
bitArray[element] = []bool{false, true}
}
}
return bitArray
}
func hashBuckets(numSets, k int) [][]int {
minHashValues := make([][]int, numSets)
for i := 0; i < numSets; i++ {
minHashValues[i] = make([]int, k)
}
for i := 0; i < numSets; i++ {
for j := 0; j < k; j++ {
minHashValues[i][j] = math.MaxInt32
}
}
return minHashValues
}
func computeHash(x, a, b, u uint) uint {
return (a*x + b) >> (32 - u)
}
func similarity(minHashValues [][]int, k int) float32 {
identicalMinHashes := 0
for i := 0; i < k; i++ {
if minHashValues[0][i] == minHashValues[1][i] {
identicalMinHashes++
}
}
return (1.0 * float32(identicalMinHashes)) / float32(k)
}

263
vendor/github.com/tylertreat/BoomFilters/partitioned.go generated vendored Normal file
View file

@ -0,0 +1,263 @@
/*
Original work Copyright (c) 2013 zhenjl
Modified work Copyright (c) 2015 Tyler Treat
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
*/
package boom
import (
"bytes"
"encoding/binary"
"hash"
"hash/fnv"
"io"
"math"
)
// PartitionedBloomFilter implements a variation of a classic Bloom filter as
// described by Almeida, Baquero, Preguica, and Hutchison in Scalable Bloom
// Filters:
//
// http://gsd.di.uminho.pt/members/cbm/ps/dbloom.pdf
//
// This filter works by partitioning the M-sized bit array into k slices of
// size m = M/k bits. Each hash function produces an index over m for its
// respective slice. Thus, each element is described by exactly k bits, meaning
// the distribution of false positives is uniform across all elements.
type PartitionedBloomFilter struct {
partitions []*Buckets // partitioned filter data
hash hash.Hash64 // hash function (kernel for all k functions)
m uint // filter size (divided into k partitions)
k uint // number of hash functions (and partitions)
s uint // partition size (m / k)
count uint // number of items added
}
// NewPartitionedBloomFilter creates a new partitioned Bloom filter optimized
// to store n items with a specified target false-positive rate.
func NewPartitionedBloomFilter(n uint, fpRate float64) *PartitionedBloomFilter {
var (
m = OptimalM(n, fpRate)
k = OptimalK(fpRate)
partitions = make([]*Buckets, k)
s = uint(math.Ceil(float64(m) / float64(k)))
)
for i := uint(0); i < k; i++ {
partitions[i] = NewBuckets(s, 1)
}
return &PartitionedBloomFilter{
partitions: partitions,
hash: fnv.New64(),
m: m,
k: k,
s: s,
}
}
// Capacity returns the Bloom filter capacity, m.
func (p *PartitionedBloomFilter) Capacity() uint {
return p.m
}
// K returns the number of hash functions.
func (p *PartitionedBloomFilter) K() uint {
return p.k
}
// Count returns the number of items added to the filter.
func (p *PartitionedBloomFilter) Count() uint {
return p.count
}
// EstimatedFillRatio returns the current estimated ratio of set bits.
func (p *PartitionedBloomFilter) EstimatedFillRatio() float64 {
return 1 - math.Exp(-float64(p.count)/float64(p.s))
}
// FillRatio returns the average ratio of set bits across all partitions.
func (p *PartitionedBloomFilter) FillRatio() float64 {
t := float64(0)
for i := uint(0); i < p.k; i++ {
sum := uint32(0)
for j := uint(0); j < p.partitions[i].Count(); j++ {
sum += p.partitions[i].Get(j)
}
t += (float64(sum) / float64(p.s))
}
return t / float64(p.k)
}
// Test will test for membership of the data and returns true if it is a
// member, false if not. This is a probabilistic test, meaning there is a
// non-zero probability of false positives but a zero probability of false
// negatives. Due to the way the filter is partitioned, the probability of
// false positives is uniformly distributed across all elements.
func (p *PartitionedBloomFilter) Test(data []byte) bool {
lower, upper := hashKernel(data, p.hash)
// If any of the K partition bits are not set, then it's not a member.
for i := uint(0); i < p.k; i++ {
if p.partitions[i].Get((uint(lower)+uint(upper)*i)%p.s) == 0 {
return false
}
}
return true
}
// Add will add the data to the Bloom filter. It returns the filter to allow
// for chaining.
func (p *PartitionedBloomFilter) Add(data []byte) Filter {
lower, upper := hashKernel(data, p.hash)
// Set the K partition bits.
for i := uint(0); i < p.k; i++ {
p.partitions[i].Set((uint(lower)+uint(upper)*i)%p.s, 1)
}
p.count++
return p
}
// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
// the data is a member, false if not.
func (p *PartitionedBloomFilter) TestAndAdd(data []byte) bool {
lower, upper := hashKernel(data, p.hash)
member := true
// If any of the K partition bits are not set, then it's not a member.
for i := uint(0); i < p.k; i++ {
idx := (uint(lower) + uint(upper)*i) % p.s
if p.partitions[i].Get(idx) == 0 {
member = false
}
p.partitions[i].Set(idx, 1)
}
p.count++
return member
}
// Reset restores the Bloom filter to its original state. It returns the filter
// to allow for chaining.
func (p *PartitionedBloomFilter) Reset() *PartitionedBloomFilter {
for _, partition := range p.partitions {
partition.Reset()
}
return p
}
// SetHash sets the hashing function used in the filter.
// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
func (p *PartitionedBloomFilter) SetHash(h hash.Hash64) {
p.hash = h
}
// WriteTo writes a binary representation of the PartitionedBloomFilter to an i/o stream.
// It returns the number of bytes written.
func (p *PartitionedBloomFilter) WriteTo(stream io.Writer) (int64, error) {
err := binary.Write(stream, binary.BigEndian, uint64(p.m))
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, uint64(p.k))
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, uint64(p.s))
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, uint64(p.count))
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, uint64(len(p.partitions)))
if err != nil {
return 0, err
}
var numBytes int64
for _, partition := range p.partitions {
num, err := partition.WriteTo(stream)
if err != nil {
return 0, err
}
numBytes += num
}
return numBytes + int64(5*binary.Size(uint64(0))), err
}
// ReadFrom reads a binary representation of PartitionedBloomFilter (such as might
// have been written by WriteTo()) from an i/o stream. It returns the number
// of bytes read.
func (p *PartitionedBloomFilter) ReadFrom(stream io.Reader) (int64, error) {
var m, k, s, count, len uint64
err := binary.Read(stream, binary.BigEndian, &m)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &k)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &s)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &count)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &len)
if err != nil {
return 0, err
}
var numBytes int64
partitions := make([]*Buckets, len)
for i, _ := range partitions {
buckets := &Buckets{}
num, err := buckets.ReadFrom(stream)
if err != nil {
return 0, err
}
numBytes += num
partitions[i] = buckets
}
p.m = uint(m)
p.k = uint(k)
p.s = uint(s)
p.count = uint(count)
p.partitions = partitions
return numBytes + int64(5*binary.Size(uint64(0))), nil
}
// GobEncode implements gob.GobEncoder interface.
func (p *PartitionedBloomFilter) GobEncode() ([]byte, error) {
var buf bytes.Buffer
_, err := p.WriteTo(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// GobDecode implements gob.GobDecoder interface.
func (p *PartitionedBloomFilter) GobDecode(data []byte) error {
buf := bytes.NewBuffer(data)
_, err := p.ReadFrom(buf)
return err
}

259
vendor/github.com/tylertreat/BoomFilters/scalable.go generated vendored Normal file
View file

@ -0,0 +1,259 @@
/*
Original work Copyright (c) 2013 zhenjl
Modified work Copyright (c) 2015 Tyler Treat
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
*/
package boom
import (
"bytes"
"encoding/binary"
"hash"
"io"
"math"
)
// ScalableBloomFilter implements a Scalable Bloom Filter as described by
// Almeida, Baquero, Preguica, and Hutchison in Scalable Bloom Filters:
//
// http://gsd.di.uminho.pt/members/cbm/ps/dbloom.pdf
//
// A Scalable Bloom Filter dynamically adapts to the number of elements in the
// data set while enforcing a tight upper bound on the false-positive rate.
// This works by adding Bloom filters with geometrically decreasing
// false-positive rates as filters become full. The tightening ratio, r,
// controls the filter growth. The compounded probability over the whole series
// converges to a target value, even accounting for an infinite series.
//
// Scalable Bloom Filters are useful for cases where the size of the data set
// isn't known a priori and memory constraints aren't of particular concern.
// For situations where memory is bounded, consider using Inverse or Stable
// Bloom Filters.
type ScalableBloomFilter struct {
filters []*PartitionedBloomFilter // filters with geometrically decreasing error rates
r float64 // tightening ratio
fp float64 // target false-positive rate
p float64 // partition fill ratio
hint uint // filter size hint
}
// NewScalableBloomFilter creates a new Scalable Bloom Filter with the
// specified target false-positive rate and tightening ratio. Use
// NewDefaultScalableBloomFilter if you don't want to calculate these
// parameters.
func NewScalableBloomFilter(hint uint, fpRate, r float64) *ScalableBloomFilter {
s := &ScalableBloomFilter{
filters: make([]*PartitionedBloomFilter, 0, 1),
r: r,
fp: fpRate,
p: fillRatio,
hint: hint,
}
s.addFilter()
return s
}
// NewDefaultScalableBloomFilter creates a new Scalable Bloom Filter with the
// specified target false-positive rate and an optimal tightening ratio.
func NewDefaultScalableBloomFilter(fpRate float64) *ScalableBloomFilter {
return NewScalableBloomFilter(10000, fpRate, 0.8)
}
// Capacity returns the current Scalable Bloom Filter capacity, which is the
// sum of the capacities for the contained series of Bloom filters.
func (s *ScalableBloomFilter) Capacity() uint {
capacity := uint(0)
for _, bf := range s.filters {
capacity += bf.Capacity()
}
return capacity
}
// K returns the number of hash functions used in each Bloom filter.
func (s *ScalableBloomFilter) K() uint {
// K is the same across every filter.
return s.filters[0].K()
}
// FillRatio returns the average ratio of set bits across every filter.
func (s *ScalableBloomFilter) FillRatio() float64 {
sum := 0.0
for _, filter := range s.filters {
sum += filter.FillRatio()
}
return sum / float64(len(s.filters))
}
// Test will test for membership of the data and returns true if it is a
// member, false if not. This is a probabilistic test, meaning there is a
// non-zero probability of false positives but a zero probability of false
// negatives.
func (s *ScalableBloomFilter) Test(data []byte) bool {
// Querying is made by testing for the presence in each filter.
for _, bf := range s.filters {
if bf.Test(data) {
return true
}
}
return false
}
// Add will add the data to the Bloom filter. It returns the filter to allow
// for chaining.
func (s *ScalableBloomFilter) Add(data []byte) Filter {
idx := len(s.filters) - 1
// If the last filter has reached its fill ratio, add a new one.
if s.filters[idx].EstimatedFillRatio() >= s.p {
s.addFilter()
idx++
}
s.filters[idx].Add(data)
return s
}
// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
// the data is a member, false if not.
func (s *ScalableBloomFilter) TestAndAdd(data []byte) bool {
member := s.Test(data)
s.Add(data)
return member
}
// Reset restores the Bloom filter to its original state. It returns the filter
// to allow for chaining.
func (s *ScalableBloomFilter) Reset() *ScalableBloomFilter {
s.filters = make([]*PartitionedBloomFilter, 0, 1)
s.addFilter()
return s
}
// addFilter adds a new Bloom filter with a restricted false-positive rate to
// the Scalable Bloom Filter
func (s *ScalableBloomFilter) addFilter() {
fpRate := s.fp * math.Pow(s.r, float64(len(s.filters)))
p := NewPartitionedBloomFilter(s.hint, fpRate)
if len(s.filters) > 0 {
p.SetHash(s.filters[0].hash)
}
s.filters = append(s.filters, p)
}
// SetHash sets the hashing function used in the filter.
// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
func (s *ScalableBloomFilter) SetHash(h hash.Hash64) {
for _, bf := range s.filters {
bf.SetHash(h)
}
}
// WriteTo writes a binary representation of the ScalableBloomFilter to an i/o stream.
// It returns the number of bytes written.
func (s *ScalableBloomFilter) WriteTo(stream io.Writer) (int64, error) {
err := binary.Write(stream, binary.BigEndian, s.r)
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, s.fp)
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, s.p)
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, uint64(s.hint))
if err != nil {
return 0, err
}
err = binary.Write(stream, binary.BigEndian, uint64(len(s.filters)))
if err != nil {
return 0, err
}
var numBytes int64
for _, filter := range s.filters {
num, err := filter.WriteTo(stream)
if err != nil {
return 0, err
}
numBytes += num
}
return numBytes + int64(5*binary.Size(uint64(0))), err
}
// ReadFrom reads a binary representation of ScalableBloomFilter (such as might
// have been written by WriteTo()) from an i/o stream. It returns the number
// of bytes read.
func (s *ScalableBloomFilter) ReadFrom(stream io.Reader) (int64, error) {
var r, fp, p float64
var hint, len uint64
err := binary.Read(stream, binary.BigEndian, &r)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &fp)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &p)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &hint)
if err != nil {
return 0, err
}
err = binary.Read(stream, binary.BigEndian, &len)
if err != nil {
return 0, err
}
var numBytes int64
filters := make([]*PartitionedBloomFilter, len)
for i, _ := range filters {
filter := NewPartitionedBloomFilter(0, fp)
num, err := filter.ReadFrom(stream)
if err != nil {
return 0, err
}
numBytes += num
filters[i] = filter
}
s.r = r
s.fp = fp
s.p = p
s.hint = uint(hint)
s.filters = filters
return numBytes + int64(5*binary.Size(uint64(0))), nil
}
// GobEncode implements gob.GobEncoder interface.
func (s *ScalableBloomFilter) GobEncode() ([]byte, error) {
var buf bytes.Buffer
_, err := s.WriteTo(&buf)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// GobDecode implements gob.GobDecoder interface.
func (s *ScalableBloomFilter) GobDecode(data []byte) error {
buf := bytes.NewBuffer(data)
_, err := s.ReadFrom(buf)
return err
}

227
vendor/github.com/tylertreat/BoomFilters/stable.go generated vendored Normal file
View file

@ -0,0 +1,227 @@
package boom
import (
"hash"
"hash/fnv"
"math"
"math/rand"
)
// StableBloomFilter implements a Stable Bloom Filter as described by Deng and
// Rafiei in Approximately Detecting Duplicates for Streaming Data using Stable
// Bloom Filters:
//
// http://webdocs.cs.ualberta.ca/~drafiei/papers/DupDet06Sigmod.pdf
//
// A Stable Bloom Filter (SBF) continuously evicts stale information so that it
// has room for more recent elements. Like traditional Bloom filters, an SBF
// has a non-zero probability of false positives, which is controlled by
// several parameters. Unlike the classic Bloom filter, an SBF has a tight
// upper bound on the rate of false positives while introducing a non-zero rate
// of false negatives. The false-positive rate of a classic Bloom filter
// eventually reaches 1, after which all queries result in a false positive.
// The stable-point property of an SBF means the false-positive rate
// asymptotically approaches a configurable fixed constant. A classic Bloom
// filter is actually a special case of SBF where the eviction rate is zero, so
// this package provides support for them as well.
//
// Stable Bloom Filters are useful for cases where the size of the data set
// isn't known a priori, which is a requirement for traditional Bloom filters,
// and memory is bounded. For example, an SBF can be used to deduplicate
// events from an unbounded event stream with a specified upper bound on false
// positives and minimal false negatives.
type StableBloomFilter struct {
cells *Buckets // filter data
hash hash.Hash64 // hash function (kernel for all k functions)
m uint // number of cells
p uint // number of cells to decrement
k uint // number of hash functions
max uint8 // cell max value
indexBuffer []uint // buffer used to cache indices
}
// NewStableBloomFilter creates a new Stable Bloom Filter with m cells and d
// bits allocated per cell optimized for the target false-positive rate. Use
// NewDefaultStableFilter if you don't want to calculate d.
func NewStableBloomFilter(m uint, d uint8, fpRate float64) *StableBloomFilter {
k := OptimalK(fpRate) / 2
if k > m {
k = m
} else if k <= 0 {
k = 1
}
cells := NewBuckets(m, d)
return &StableBloomFilter{
hash: fnv.New64(),
m: m,
k: k,
p: optimalStableP(m, k, d, fpRate),
max: cells.MaxBucketValue(),
cells: cells,
indexBuffer: make([]uint, k),
}
}
// NewDefaultStableBloomFilter creates a new Stable Bloom Filter with m 1-bit
// cells and which is optimized for cases where there is no prior knowledge of
// the input data stream while maintaining an upper bound using the provided
// rate of false positives.
func NewDefaultStableBloomFilter(m uint, fpRate float64) *StableBloomFilter {
return NewStableBloomFilter(m, 1, fpRate)
}
// NewUnstableBloomFilter creates a new special case of Stable Bloom Filter
// which is a traditional Bloom filter with m bits and an optimal number of
// hash functions for the target false-positive rate. Unlike the stable
// variant, data is not evicted and a cell contains a maximum of 1 hash value.
func NewUnstableBloomFilter(m uint, fpRate float64) *StableBloomFilter {
var (
cells = NewBuckets(m, 1)
k = OptimalK(fpRate)
)
return &StableBloomFilter{
hash: fnv.New64(),
m: m,
k: k,
p: 0,
max: cells.MaxBucketValue(),
cells: cells,
indexBuffer: make([]uint, k),
}
}
// Cells returns the number of cells in the Stable Bloom Filter.
func (s *StableBloomFilter) Cells() uint {
return s.m
}
// K returns the number of hash functions.
func (s *StableBloomFilter) K() uint {
return s.k
}
// P returns the number of cells decremented on every add.
func (s *StableBloomFilter) P() uint {
return s.p
}
// StablePoint returns the limit of the expected fraction of zeros in the
// Stable Bloom Filter when the number of iterations goes to infinity. When
// this limit is reached, the Stable Bloom Filter is considered stable.
func (s *StableBloomFilter) StablePoint() float64 {
var (
subDenom = float64(s.p) * (1/float64(s.k) - 1/float64(s.m))
denom = 1 + 1/subDenom
base = 1 / denom
)
return math.Pow(base, float64(s.max))
}
// FalsePositiveRate returns the upper bound on false positives when the filter
// has become stable.
func (s *StableBloomFilter) FalsePositiveRate() float64 {
return math.Pow(1-s.StablePoint(), float64(s.k))
}
// Test will test for membership of the data and returns true if it is a
// member, false if not. This is a probabilistic test, meaning there is a
// non-zero probability of false positives and false negatives.
func (s *StableBloomFilter) Test(data []byte) bool {
lower, upper := hashKernel(data, s.hash)
// If any of the K cells are 0, then it's not a member.
for i := uint(0); i < s.k; i++ {
if s.cells.Get((uint(lower)+uint(upper)*i)%s.m) == 0 {
return false
}
}
return true
}
// Add will add the data to the Stable Bloom Filter. It returns the filter to
// allow for chaining.
func (s *StableBloomFilter) Add(data []byte) Filter {
// Randomly decrement p cells to make room for new elements.
s.decrement()
lower, upper := hashKernel(data, s.hash)
// Set the K cells to max.
for i := uint(0); i < s.k; i++ {
s.cells.Set((uint(lower)+uint(upper)*i)%s.m, s.max)
}
return s
}
// TestAndAdd is equivalent to calling Test followed by Add. It returns true if
// the data is a member, false if not.
func (s *StableBloomFilter) TestAndAdd(data []byte) bool {
lower, upper := hashKernel(data, s.hash)
member := true
// If any of the K cells are 0, then it's not a member.
for i := uint(0); i < s.k; i++ {
s.indexBuffer[i] = (uint(lower) + uint(upper)*i) % s.m
if s.cells.Get(s.indexBuffer[i]) == 0 {
member = false
}
}
// Randomly decrement p cells to make room for new elements.
s.decrement()
// Set the K cells to max.
for _, idx := range s.indexBuffer {
s.cells.Set(idx, s.max)
}
return member
}
// Reset restores the Stable Bloom Filter to its original state. It returns the
// filter to allow for chaining.
func (s *StableBloomFilter) Reset() *StableBloomFilter {
s.cells.Reset()
return s
}
// decrement will decrement a random cell and (p-1) adjacent cells by 1. This
// is faster than generating p random numbers. Although the processes of
// picking the p cells are not independent, each cell has a probability of p/m
// for being picked at each iteration, which means the properties still hold.
func (s *StableBloomFilter) decrement() {
r := rand.Intn(int(s.m))
for i := uint(0); i < s.p; i++ {
idx := (r + int(i)) % int(s.m)
s.cells.Increment(uint(idx), -1)
}
}
// SetHash sets the hashing function used in the filter.
// For the effect on false positive rates see: https://github.com/tylertreat/BoomFilters/pull/1
func (s *StableBloomFilter) SetHash(h hash.Hash64) {
s.hash = h
}
// optimalStableP returns the optimal number of cells to decrement, p, per
// iteration for the provided parameters of an SBF.
func optimalStableP(m, k uint, d uint8, fpRate float64) uint {
var (
max = math.Pow(2, float64(d)) - 1
subDenom = math.Pow(1-math.Pow(fpRate, 1/float64(k)), 1/max)
denom = (1/subDenom - 1) * (1/float64(k) - 1/float64(m))
)
p := uint(1 / denom)
if p <= 0 {
p = 1
}
return p
}

125
vendor/github.com/tylertreat/BoomFilters/topk.go generated vendored Normal file
View file

@ -0,0 +1,125 @@
package boom
import (
"bytes"
"container/heap"
)
type Element struct {
Data []byte
Freq uint64
}
// An elementHeap is a min-heap of elements.
type elementHeap []*Element
func (e elementHeap) Len() int { return len(e) }
func (e elementHeap) Less(i, j int) bool { return e[i].Freq < e[j].Freq }
func (e elementHeap) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
func (e *elementHeap) Push(x interface{}) {
*e = append(*e, x.(*Element))
}
func (e *elementHeap) Pop() interface{} {
old := *e
n := len(old)
x := old[n-1]
*e = old[0 : n-1]
return x
}
// TopK uses a Count-Min Sketch to calculate the top-K frequent elements in a
// stream.
type TopK struct {
cms *CountMinSketch
k uint
n uint
elements *elementHeap
}
// NewTopK creates a new TopK backed by a Count-Min sketch whose relative
// accuracy is within a factor of epsilon with probability delta. It tracks the
// k-most frequent elements.
func NewTopK(epsilon, delta float64, k uint) *TopK {
elements := make(elementHeap, 0, k)
heap.Init(&elements)
return &TopK{
cms: NewCountMinSketch(epsilon, delta),
k: k,
elements: &elements,
}
}
// Add will add the data to the Count-Min Sketch and update the top-k heap if
// applicable. Returns the TopK to allow for chaining.
func (t *TopK) Add(data []byte) *TopK {
t.cms.Add(data)
t.n++
freq := t.cms.Count(data)
if t.isTop(freq) {
t.insert(data, freq)
}
return t
}
// Elements returns the top-k elements from lowest to highest frequency.
func (t *TopK) Elements() []*Element {
if t.elements.Len() == 0 {
return make([]*Element, 0)
}
elements := make(elementHeap, t.elements.Len())
copy(elements, *t.elements)
heap.Init(&elements)
topK := make([]*Element, 0, t.k)
for elements.Len() > 0 {
topK = append(topK, heap.Pop(&elements).(*Element))
}
return topK
}
// Reset restores the TopK to its original state. It returns itself to allow
// for chaining.
func (t *TopK) Reset() *TopK {
t.cms.Reset()
elements := make(elementHeap, 0, t.k)
heap.Init(&elements)
t.elements = &elements
t.n = 0
return t
}
// isTop indicates if the given frequency falls within the top-k heap.
func (t *TopK) isTop(freq uint64) bool {
if t.elements.Len() < int(t.k) {
return true
}
return freq >= (*t.elements)[0].Freq
}
// insert adds the data to the top-k heap. If the data is already an element,
// the frequency is updated. If the heap already has k elements, the element
// with the minimum frequency is removed.
func (t *TopK) insert(data []byte, freq uint64) {
for _, element := range *t.elements {
if bytes.Compare(data, element.Data) == 0 {
// Element already in top-k.
element.Freq = freq
return
}
}
if t.elements.Len() == int(t.k) {
// Remove minimum-frequency element.
heap.Pop(t.elements)
}
// Add element to top-k.
heap.Push(t.elements, &Element{Data: data, Freq: freq})
}