eth/filters: removed mip map filtering, always using bloombits

This commit is contained in:
Zsolt Felfoldi 2017-04-19 18:34:59 +02:00
parent e860278a96
commit d85fb130bd
11 changed files with 27 additions and 402 deletions

View file

@ -783,12 +783,6 @@ func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain
log.Crit("Failed to write block receipts", "err", err) log.Crit("Failed to write block receipts", "err", err)
return return
} }
if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
errs[index] = fmt.Errorf("failed to write log blooms: %v", err)
atomic.AddInt32(&failed, 1)
log.Crit("Failed to write log blooms", "err", err)
return
}
if err := WriteTransactions(self.chainDb, block); err != nil { if err := WriteTransactions(self.chainDb, block); err != nil {
errs[index] = fmt.Errorf("failed to write individual transactions: %v", err) errs[index] = fmt.Errorf("failed to write individual transactions: %v", err)
atomic.AddInt32(&failed, 1) atomic.AddInt32(&failed, 1)
@ -1041,10 +1035,6 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
if err := WriteReceipts(self.chainDb, receipts); err != nil { if err := WriteReceipts(self.chainDb, receipts); err != nil {
return i, err return i, err
} }
// Write map map bloom filters
if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
return i, err
}
// Write hash preimages // Write hash preimages
if err := WritePreimages(self.chainDb, block.NumberU64(), self.stateCache.Preimages()); err != nil { if err := WritePreimages(self.chainDb, block.NumberU64(), self.stateCache.Preimages()); err != nil {
return i, err return i, err
@ -1207,10 +1197,6 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
if err := WriteReceipts(self.chainDb, receipts); err != nil { if err := WriteReceipts(self.chainDb, receipts); err != nil {
return err return err
} }
// Write map map bloom filters
if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
return err
}
addedTxs = append(addedTxs, block.Transactions()...) addedTxs = append(addedTxs, block.Transactions()...)
} }

View file

@ -23,7 +23,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"math/big" "math/big"
"sync"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/bloombits"
@ -51,9 +50,6 @@ var (
txMetaSuffix = []byte{0x01} txMetaSuffix = []byte{0x01}
receiptsPrefix = []byte("receipts-") receiptsPrefix = []byte("receipts-")
mipmapPre = []byte("mipmap-log-bloom-")
MIPMapLevels = []uint64{1000000, 500000, 100000, 50000, 1000}
configPrefix = []byte("ethereum-config-") // config prefix for the db configPrefix = []byte("ethereum-config-") // config prefix for the db
// used by old (non-sequential keys) db, now only used for conversion // used by old (non-sequential keys) db, now only used for conversion
@ -67,8 +63,6 @@ var (
ChainConfigNotFoundErr = errors.New("ChainConfig not found") // general config not found error ChainConfigNotFoundErr = errors.New("ChainConfig not found") // general config not found error
mipmapBloomMu sync.Mutex // protect against race condition when updating mipmap blooms
preimageCounter = metrics.NewCounter("db/preimage/total") preimageCounter = metrics.NewCounter("db/preimage/total")
preimageHitCounter = metrics.NewCounter("db/preimage/hits") preimageHitCounter = metrics.NewCounter("db/preimage/hits")
@ -539,48 +533,6 @@ func DeleteReceipt(db ethdb.Database, hash common.Hash) {
db.Delete(append(receiptsPrefix, hash.Bytes()...)) db.Delete(append(receiptsPrefix, hash.Bytes()...))
} }
// returns a formatted MIP mapped key by adding prefix, canonical number and level
//
// ex. fn(98, 1000) = (prefix || 1000 || 0)
func mipmapKey(num, level uint64) []byte {
lkey := make([]byte, 8)
binary.BigEndian.PutUint64(lkey, level)
key := new(big.Int).SetUint64(num / level * level)
return append(mipmapPre, append(lkey, key.Bytes()...)...)
}
// WriteMapmapBloom writes each address included in the receipts' logs to the
// MIP bloom bin.
func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts) error {
mipmapBloomMu.Lock()
defer mipmapBloomMu.Unlock()
batch := db.NewBatch()
for _, level := range MIPMapLevels {
key := mipmapKey(number, level)
bloomDat, _ := db.Get(key)
bloom := types.BytesToBloom(bloomDat)
for _, receipt := range receipts {
for _, log := range receipt.Logs {
bloom.Add(log.Address.Big())
}
}
batch.Put(key, bloom.Bytes())
}
if err := batch.Write(); err != nil {
return fmt.Errorf("mipmap write fail for: %d: %v", number, err)
}
return nil
}
// GetMipmapBloom returns a bloom filter using the number and level as input
// parameters. For available levels see MIPMapLevels.
func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom {
bloomDat, _ := db.Get(mipmapKey(number, level))
return types.BytesToBloom(bloomDat)
}
// PreimageTable returns a Database instance with the key prefix for preimage entries. // PreimageTable returns a Database instance with the key prefix for preimage entries.
func PreimageTable(db ethdb.Database) ethdb.Database { func PreimageTable(db ethdb.Database) ethdb.Database {
return ethdb.NewTable(db, preimagePrefix) return ethdb.NewTable(db, preimagePrefix)

View file

@ -18,17 +18,13 @@ package core
import ( import (
"bytes" "bytes"
"io/ioutil"
"math/big" "math/big"
"os"
"testing" "testing"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
) )
@ -446,111 +442,3 @@ func TestBlockReceiptStorage(t *testing.T) {
t.Fatalf("deleted receipts returned: %v", rs) t.Fatalf("deleted receipts returned: %v", rs)
} }
} }
func TestMipmapBloom(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
receipt1 := new(types.Receipt)
receipt1.Logs = []*types.Log{
{Address: common.BytesToAddress([]byte("test"))},
{Address: common.BytesToAddress([]byte("address"))},
}
receipt2 := new(types.Receipt)
receipt2.Logs = []*types.Log{
{Address: common.BytesToAddress([]byte("test"))},
{Address: common.BytesToAddress([]byte("address1"))},
}
WriteMipmapBloom(db, 1, types.Receipts{receipt1})
WriteMipmapBloom(db, 2, types.Receipts{receipt2})
for _, level := range MIPMapLevels {
bloom := GetMipmapBloom(db, 2, level)
if !bloom.Test(new(big.Int).SetBytes([]byte("address1"))) {
t.Error("expected test to be included on level:", level)
}
}
// reset
db, _ = ethdb.NewMemDatabase()
receipt := new(types.Receipt)
receipt.Logs = []*types.Log{
{Address: common.BytesToAddress([]byte("test"))},
}
WriteMipmapBloom(db, 999, types.Receipts{receipt1})
receipt = new(types.Receipt)
receipt.Logs = []*types.Log{
{Address: common.BytesToAddress([]byte("test 1"))},
}
WriteMipmapBloom(db, 1000, types.Receipts{receipt})
bloom := GetMipmapBloom(db, 1000, 1000)
if bloom.TestBytes([]byte("test")) {
t.Error("test should not have been included")
}
}
func TestMipmapChain(t *testing.T) {
dir, err := ioutil.TempDir("", "mipmap")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
var (
db, _ = ethdb.NewLDBDatabase(dir, 0, 0)
key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
addr = crypto.PubkeyToAddress(key1.PublicKey)
addr2 = common.BytesToAddress([]byte("jeff"))
hash1 = common.BytesToHash([]byte("topic1"))
)
defer db.Close()
gspec := &Genesis{
Config: params.TestChainConfig,
Alloc: GenesisAlloc{addr: {Balance: big.NewInt(1000000)}},
}
genesis := gspec.MustCommit(db)
chain, receipts := GenerateChain(params.TestChainConfig, genesis, db, 1010, func(i int, gen *BlockGen) {
var receipts types.Receipts
switch i {
case 1:
receipt := types.NewReceipt(nil, new(big.Int))
receipt.Logs = []*types.Log{{Address: addr, Topics: []common.Hash{hash1}}}
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
case 1000:
receipt := types.NewReceipt(nil, new(big.Int))
receipt.Logs = []*types.Log{{Address: addr2}}
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
}
// store the receipts
err := WriteReceipts(db, receipts)
if err != nil {
t.Fatal(err)
}
WriteMipmapBloom(db, uint64(i+1), receipts)
})
for i, block := range chain {
WriteBlock(db, block)
if err := WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil {
t.Fatalf("failed to insert block number: %v", err)
}
if err := WriteHeadBlockHash(db, block.Hash()); err != nil {
t.Fatalf("failed to insert block number: %v", err)
}
if err := WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil {
t.Fatal("error writing block receipts:", err)
}
}
bloom := GetMipmapBloom(db, 0, 1000)
if bloom.TestBytes(addr2[:]) {
t.Error("address was included in bloom and should not have")
}
}

View file

@ -123,9 +123,6 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
MinerThreads: config.MinerThreads, MinerThreads: config.MinerThreads,
} }
if err := addMipmapBloomBins(chainDb); err != nil {
return nil, err
}
log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId)
if !config.SkipBcVersionCheck { if !config.SkipBcVersionCheck {
@ -239,10 +236,6 @@ func (s *Ethereum) APIs() []rpc.API {
// Append any APIs exposed explicitly by the consensus engine // Append any APIs exposed explicitly by the consensus engine
apis = append(apis, s.engine.APIs(s.BlockChain())...) apis = append(apis, s.engine.APIs(s.BlockChain())...)
var bbSection uint64
if useBloomBits {
bbSection = bloomBitsSection
}
// Append all the local APIs and return // Append all the local APIs and return
return append(apis, []rpc.API{ return append(apis, []rpc.API{
{ {
@ -268,7 +261,7 @@ func (s *Ethereum) APIs() []rpc.API {
}, { }, {
Namespace: "eth", Namespace: "eth",
Version: "1.0", Version: "1.0",
Service: filters.NewPublicFilterAPI(s.ApiBackend, false, bbSection), Service: filters.NewPublicFilterAPI(s.ApiBackend, false, bloomBitsSection),
Public: true, Public: true,
}, { }, {
Namespace: "admin", Namespace: "admin",

View file

@ -1,83 +0,0 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package eth
import (
"math/big"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
)
func TestMipmapUpgrade(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
addr := common.BytesToAddress([]byte("jeff"))
genesis := new(core.Genesis).MustCommit(db)
chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, db, 10, func(i int, gen *core.BlockGen) {
var receipts types.Receipts
switch i {
case 1:
receipt := types.NewReceipt(nil, new(big.Int))
receipt.Logs = []*types.Log{{Address: addr}}
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
case 2:
receipt := types.NewReceipt(nil, new(big.Int))
receipt.Logs = []*types.Log{{Address: addr}}
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
}
// store the receipts
err := core.WriteReceipts(db, receipts)
if err != nil {
t.Fatal(err)
}
})
for i, block := range chain {
core.WriteBlock(db, block)
if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil {
t.Fatalf("failed to insert block number: %v", err)
}
if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil {
t.Fatalf("failed to insert block number: %v", err)
}
if err := core.WriteBlockReceipts(db, block.Hash(), block.NumberU64(), receipts[i]); err != nil {
t.Fatal("error writing block receipts:", err)
}
}
err := addMipmapBloomBins(db)
if err != nil {
t.Fatal(err)
}
bloom := core.GetMipmapBloom(db, 1, core.MIPMapLevels[0])
if (bloom == types.Bloom{}) {
t.Error("got empty bloom filter")
}
data, _ := db.Get([]byte("setting-mipmap-version"))
if len(data) == 0 {
t.Error("setting-mipmap-version not written to database")
}
}

View file

@ -20,11 +20,9 @@ package eth
import ( import (
"bytes" "bytes"
"encoding/binary" "encoding/binary"
"fmt"
"math/big" "math/big"
"time" "time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
@ -253,49 +251,6 @@ func upgradeSequentialBlockData(db ethdb.Database, hash []byte) error {
return nil return nil
} }
func addMipmapBloomBins(db ethdb.Database) (err error) {
const mipmapVersion uint = 2
// check if the version is set. We ignore data for now since there's
// only one version so we can easily ignore it for now
var data []byte
data, _ = db.Get([]byte("setting-mipmap-version"))
if len(data) > 0 {
var version uint
if err := rlp.DecodeBytes(data, &version); err == nil && version == mipmapVersion {
return nil
}
}
defer func() {
if err == nil {
var val []byte
val, err = rlp.EncodeToBytes(mipmapVersion)
if err == nil {
err = db.Put([]byte("setting-mipmap-version"), val)
}
return
}
}()
latestHash := core.GetHeadBlockHash(db)
latestBlock := core.GetBlock(db, latestHash, core.GetBlockNumber(db, latestHash))
if latestBlock == nil { // clean database
return
}
tstart := time.Now()
log.Warn("Upgrading db log bloom bins")
for i := uint64(0); i <= latestBlock.NumberU64(); i++ {
hash := core.GetCanonicalHash(db, i)
if (hash == common.Hash{}) {
return fmt.Errorf("chain db corrupted. Could not find block %d.", i)
}
core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash, i))
}
log.Info("Bloom-bin upgrade completed", "elapsed", common.PrettyDuration(time.Since(tstart)))
return nil
}
// BloomBitsProcessorBackend implements ChainSectionProcessorBackend // BloomBitsProcessorBackend implements ChainSectionProcessorBackend
type BloomBitsProcessorBackend struct { type BloomBitsProcessorBackend struct {
db ethdb.Database db ethdb.Database

View file

@ -52,7 +52,6 @@ type filter struct {
// information related to the Ethereum protocol such als blocks, transactions and logs. // information related to the Ethereum protocol such als blocks, transactions and logs.
type PublicFilterAPI struct { type PublicFilterAPI struct {
backend Backend backend Backend
useMipMap bool
bloomBitsSection uint64 bloomBitsSection uint64
mux *event.TypeMux mux *event.TypeMux
quit chan struct{} quit chan struct{}
@ -66,7 +65,6 @@ type PublicFilterAPI struct {
func NewPublicFilterAPI(backend Backend, lightMode bool, bloomBitsSection uint64) *PublicFilterAPI { func NewPublicFilterAPI(backend Backend, lightMode bool, bloomBitsSection uint64) *PublicFilterAPI {
api := &PublicFilterAPI{ api := &PublicFilterAPI{
backend: backend, backend: backend,
useMipMap: bloomBitsSection == 0,
bloomBitsSection: bloomBitsSection, bloomBitsSection: bloomBitsSection,
mux: backend.EventMux(), mux: backend.EventMux(),
chainDb: backend.ChainDb(), chainDb: backend.ChainDb(),
@ -335,7 +333,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([
crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64()) crit.ToBlock = big.NewInt(rpc.LatestBlockNumber.Int64())
} }
filter := New(api.backend, api.useMipMap, api.bloomBitsSection) filter := New(api.backend, api.bloomBitsSection)
filter.SetBeginBlock(crit.FromBlock.Int64()) filter.SetBeginBlock(crit.FromBlock.Int64())
filter.SetEndBlock(crit.ToBlock.Int64()) filter.SetEndBlock(crit.ToBlock.Int64())
filter.SetAddresses(crit.Addresses) filter.SetAddresses(crit.Addresses)
@ -375,7 +373,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty
return nil, fmt.Errorf("filter not found") return nil, fmt.Errorf("filter not found")
} }
filter := New(api.backend, api.useMipMap, api.bloomBitsSection) filter := New(api.backend, api.bloomBitsSection)
if f.crit.FromBlock != nil { if f.crit.FromBlock != nil {
filter.SetBeginBlock(f.crit.FromBlock.Int64()) filter.SetBeginBlock(f.crit.FromBlock.Int64())
} else { } else {

View file

@ -18,7 +18,6 @@ package filters
import ( import (
"context" "context"
"math"
"math/big" "math/big"
"time" "time"
@ -42,7 +41,6 @@ type Backend interface {
// Filter can be used to retrieve and filter logs. // Filter can be used to retrieve and filter logs.
type Filter struct { type Filter struct {
backend Backend backend Backend
useMipMap, useBloomBits bool
bloomBitsSection uint64 bloomBitsSection uint64
created time.Time created time.Time
@ -57,13 +55,9 @@ type Filter struct {
// New creates a new filter which uses a bloom filter on blocks to figure out whether // New creates a new filter which uses a bloom filter on blocks to figure out whether
// a particular block is interesting or not. // a particular block is interesting or not.
// MipMaps allow past blocks to be searched much more efficiently, but are not available func New(backend Backend, bloomBitsSection uint64) *Filter {
// to light clients.
func New(backend Backend, useMipMap bool, bloomBitsSection uint64) *Filter {
return &Filter{ return &Filter{
backend: backend, backend: backend,
useMipMap: useMipMap,
useBloomBits: bloomBitsSection != 0,
bloomBitsSection: bloomBitsSection, bloomBitsSection: bloomBitsSection,
db: backend.ChainDb(), db: backend.ChainDb(),
matcher: bloombits.NewMatcher(bloomBitsSection), matcher: bloombits.NewMatcher(bloomBitsSection),
@ -86,17 +80,13 @@ func (f *Filter) SetEndBlock(end int64) {
// in the given addresses. // in the given addresses.
func (f *Filter) SetAddresses(addr []common.Address) { func (f *Filter) SetAddresses(addr []common.Address) {
f.addresses = addr f.addresses = addr
if f.useBloomBits {
f.matcher.SetAddresses(addr) f.matcher.SetAddresses(addr)
}
} }
// SetTopics matches only logs that have topics matching the given topics. // SetTopics matches only logs that have topics matching the given topics.
func (f *Filter) SetTopics(topics [][]common.Hash) { func (f *Filter) SetTopics(topics [][]common.Hash) {
f.topics = topics f.topics = topics
if f.useBloomBits {
f.matcher.SetTopics(topics) f.matcher.SetTopics(topics)
}
} }
// FindOnce searches the blockchain for matching log entries, returning // FindOnce searches the blockchain for matching log entries, returning
@ -119,18 +109,9 @@ func (f *Filter) FindOnce(ctx context.Context) ([]*types.Log, error) {
endBlockNo = headBlockNumber endBlockNo = headBlockNumber
} }
// if no addresses are present we can't make use of fast search which
// uses the mipmap bloom filters to check for fast inclusion and uses
// higher range probability in order to ensure at least a false positive
if !f.useMipMap || len(f.addresses) == 0 {
logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo) logs, blockNumber, err := f.getLogs(ctx, beginBlockNo, endBlockNo)
f.begin = int64(blockNumber + 1) f.begin = int64(blockNumber + 1)
return logs, err return logs, err
}
logs, blockNumber := f.mipFind(beginBlockNo, endBlockNo, 0)
f.begin = int64(blockNumber + 1)
return logs, nil
} }
// Run filters logs with the current parameters set // Run filters logs with the current parameters set
@ -144,42 +125,6 @@ func (f *Filter) Find(ctx context.Context) (logs []*types.Log, err error) {
} }
} }
func (f *Filter) mipFind(start, end uint64, depth int) (logs []*types.Log, blockNumber uint64) {
level := core.MIPMapLevels[depth]
// normalise numerator so we can work in level specific batches and
// work with the proper range checks
for num := start / level * level; num <= end; num += level {
// find addresses in bloom filters
bloom := core.GetMipmapBloom(f.db, num, level)
// Don't bother checking the first time through the loop - we're probably picking
// up where a previous run left off.
first := true
for _, addr := range f.addresses {
if first || bloom.TestBytes(addr[:]) {
first = false
// range check normalised values and make sure that
// we're resolving the correct range instead of the
// normalised values.
start := uint64(math.Max(float64(num), float64(start)))
end := uint64(math.Min(float64(num+level-1), float64(end)))
if depth+1 == len(core.MIPMapLevels) {
l, blockNumber, _ := f.getLogs(context.Background(), start, end)
if len(l) > 0 {
return l, blockNumber
}
} else {
l, blockNumber := f.mipFind(start, end, depth+1)
if len(l) > 0 {
return l, blockNumber
}
}
}
}
}
return nil, end
}
// serveMatcher serves the bloomBits matcher by fetching the requested vectors // serveMatcher serves the bloomBits matcher by fetching the requested vectors
// through the filter backend // through the filter backend
func (f *Filter) serveMatcher(ctx context.Context, stop chan struct{}) chan error { func (f *Filter) serveMatcher(ctx context.Context, stop chan struct{}) chan error {
@ -227,8 +172,8 @@ func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types.
return nil, i, nil return nil, i, nil
} }
if f.useBloomBits {
haveBloomBitsBefore := core.GetBloomBitsAvailable(f.db) * f.bloomBitsSection haveBloomBitsBefore := core.GetBloomBitsAvailable(f.db) * f.bloomBitsSection
if haveBloomBitsBefore > start {
e := end e := end
if haveBloomBitsBefore <= e { if haveBloomBitsBefore <= e {
e = haveBloomBitsBefore - 1 e = haveBloomBitsBefore - 1
@ -236,7 +181,6 @@ func (f *Filter) getLogs(ctx context.Context, start, end uint64) (logs []*types.
stop := make(chan struct{}) stop := make(chan struct{})
defer close(stop) defer close(stop)
//fmt.Println("GetMatches")
matches := f.matcher.GetMatches(start, e, stop) matches := f.matcher.GetMatches(start, e, stop)
errChn := f.serveMatcher(ctx, stop) errChn := f.serveMatcher(ctx, stop)

View file

@ -32,6 +32,8 @@ import (
"github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params"
) )
const testBloomBitsSection = 4096
func makeReceipt(addr common.Address) *types.Receipt { func makeReceipt(addr common.Address) *types.Receipt {
receipt := types.NewReceipt(nil, new(big.Int)) receipt := types.NewReceipt(nil, new(big.Int))
receipt.Logs = []*types.Log{ receipt.Logs = []*types.Log{
@ -41,8 +43,8 @@ func makeReceipt(addr common.Address) *types.Receipt {
return receipt return receipt
} }
func BenchmarkMipmaps(b *testing.B) { func BenchmarkFilters(b *testing.B) {
dir, err := ioutil.TempDir("", "mipmap") dir, err := ioutil.TempDir("", "filtertest")
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
@ -88,7 +90,6 @@ func BenchmarkMipmaps(b *testing.B) {
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
core.WriteMipmapBloom(db, uint64(i+1), receipts)
}) })
for i, block := range chain { for i, block := range chain {
core.WriteBlock(db, block) core.WriteBlock(db, block)
@ -104,7 +105,7 @@ func BenchmarkMipmaps(b *testing.B) {
} }
b.ResetTimer() b.ResetTimer()
filter := New(backend, true, 0) filter := New(backend, testBloomBitsSection)
filter.SetAddresses([]common.Address{addr1, addr2, addr3, addr4}) filter.SetAddresses([]common.Address{addr1, addr2, addr3, addr4})
filter.SetBeginBlock(0) filter.SetBeginBlock(0)
filter.SetEndBlock(-1) filter.SetEndBlock(-1)
@ -118,7 +119,7 @@ func BenchmarkMipmaps(b *testing.B) {
} }
func TestFilters(t *testing.T) { func TestFilters(t *testing.T) {
dir, err := ioutil.TempDir("", "mipmap") dir, err := ioutil.TempDir("", "filtertest")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -189,10 +190,6 @@ func TestFilters(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
// i is used as block number for the writes but since the i
// starts at 0 and block 0 (genesis) is already present increment
// by one
core.WriteMipmapBloom(db, uint64(i+1), receipts)
}) })
for i, block := range chain { for i, block := range chain {
core.WriteBlock(db, block) core.WriteBlock(db, block)
@ -207,7 +204,7 @@ func TestFilters(t *testing.T) {
} }
} }
filter := New(backend, true, 0) filter := New(backend, testBloomBitsSection)
filter.SetAddresses([]common.Address{addr}) filter.SetAddresses([]common.Address{addr})
filter.SetTopics([][]common.Hash{{hash1, hash2, hash3, hash4}}) filter.SetTopics([][]common.Hash{{hash1, hash2, hash3, hash4}})
filter.SetBeginBlock(0) filter.SetBeginBlock(0)
@ -218,7 +215,7 @@ func TestFilters(t *testing.T) {
t.Error("expected 4 log, got", len(logs)) t.Error("expected 4 log, got", len(logs))
} }
filter = New(backend, true, 0) filter = New(backend, testBloomBitsSection)
filter.SetAddresses([]common.Address{addr}) filter.SetAddresses([]common.Address{addr})
filter.SetTopics([][]common.Hash{{hash3}}) filter.SetTopics([][]common.Hash{{hash3}})
filter.SetBeginBlock(900) filter.SetBeginBlock(900)
@ -231,7 +228,7 @@ func TestFilters(t *testing.T) {
t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0])
} }
filter = New(backend, true, 0) filter = New(backend, testBloomBitsSection)
filter.SetAddresses([]common.Address{addr}) filter.SetAddresses([]common.Address{addr})
filter.SetTopics([][]common.Hash{{hash3}}) filter.SetTopics([][]common.Hash{{hash3}})
filter.SetBeginBlock(990) filter.SetBeginBlock(990)
@ -244,7 +241,7 @@ func TestFilters(t *testing.T) {
t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0])
} }
filter = New(backend, true, 0) filter = New(backend, testBloomBitsSection)
filter.SetTopics([][]common.Hash{{hash1, hash2}}) filter.SetTopics([][]common.Hash{{hash1, hash2}})
filter.SetBeginBlock(1) filter.SetBeginBlock(1)
filter.SetEndBlock(10) filter.SetEndBlock(10)
@ -255,7 +252,7 @@ func TestFilters(t *testing.T) {
} }
failHash := common.BytesToHash([]byte("fail")) failHash := common.BytesToHash([]byte("fail"))
filter = New(backend, true, 0) filter = New(backend, testBloomBitsSection)
filter.SetTopics([][]common.Hash{{failHash}}) filter.SetTopics([][]common.Hash{{failHash}})
filter.SetBeginBlock(0) filter.SetBeginBlock(0)
filter.SetEndBlock(-1) filter.SetEndBlock(-1)
@ -266,7 +263,7 @@ func TestFilters(t *testing.T) {
} }
failAddr := common.BytesToAddress([]byte("failmenow")) failAddr := common.BytesToAddress([]byte("failmenow"))
filter = New(backend, true, 0) filter = New(backend, testBloomBitsSection)
filter.SetAddresses([]common.Address{failAddr}) filter.SetAddresses([]common.Address{failAddr})
filter.SetBeginBlock(0) filter.SetBeginBlock(0)
filter.SetEndBlock(-1) filter.SetEndBlock(-1)
@ -276,7 +273,7 @@ func TestFilters(t *testing.T) {
t.Error("expected 0 log, got", len(logs)) t.Error("expected 0 log, got", len(logs))
} }
filter = New(backend, true, 0) filter = New(backend, testBloomBitsSection)
filter.SetTopics([][]common.Hash{{failHash}, {hash1}}) filter.SetTopics([][]common.Hash{{failHash}, {hash1}})
filter.SetBeginBlock(0) filter.SetBeginBlock(0)
filter.SetEndBlock(-1) filter.SetEndBlock(-1)

View file

@ -46,7 +46,6 @@ const (
softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data. softResponseLimit = 2 * 1024 * 1024 // Target maximum size of returned blocks, headers or node data.
estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header
useBloomBits = false
bloomBitsSection = 4096 bloomBitsSection = 4096
) )
@ -117,10 +116,8 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne
quitSync: make(chan struct{}), quitSync: make(chan struct{}),
} }
if useBloomBits {
manager.bloomBitsProcessor = NewBloomBitsProcessor(manager.chaindb, manager.quitSync) manager.bloomBitsProcessor = NewBloomBitsProcessor(manager.chaindb, manager.quitSync)
blockchain.AddChainProcessor(manager.bloomBitsProcessor) blockchain.AddChainProcessor(manager.bloomBitsProcessor)
}
// Figure out whether to allow fast sync or not // Figure out whether to allow fast sync or not
if mode == downloader.FastSync && blockchain.CurrentBlock().NumberU64() > 0 { if mode == downloader.FastSync && blockchain.CurrentBlock().NumberU64() > 0 {

View file

@ -300,8 +300,6 @@ func (self *worker) wait() {
core.WriteTransactions(self.chainDb, block) core.WriteTransactions(self.chainDb, block)
// store the receipts // store the receipts
core.WriteReceipts(self.chainDb, work.receipts) core.WriteReceipts(self.chainDb, work.receipts)
// Write map map bloom filters
core.WriteMipmapBloom(self.chainDb, block.NumberU64(), work.receipts)
// implicit by posting ChainHeadEvent // implicit by posting ChainHeadEvent
mustCommitNewWork = false mustCommitNewWork = false
} }