forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-27 15:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

core/bloombits: removed bit vector aliases, using bitutil functions

Browse source
This commit is contained in:
Zsolt Felfoldi 2017-05-09 00:38:37 +02:00
parent d85fb130bd
commit 3dcccfb27c
10 changed files with 49 additions and 167 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
core/bloombits/fetcher_test.go
View file

@ -27,8 +27,8 @@ import (
const testFetcherReqCnt = 50000
func fetcherTestVector(b uint, s uint64) BitVector {
r := make(BitVector, 10)
func fetcherTestVector(b uint, s uint64) []byte {
r := make([]byte, 10)
binary.BigEndian.PutUint16(r[0:2], uint16(b))
binary.BigEndian.PutUint64(r[2:10], s)
return r
@ -59,7 +59,7 @@ func testFetcher(t *testing.T, cnt int) {
}
time.Sleep(time.Duration(rand.Intn(1000000)))
atomic.AddUint32(&reqCnt, 1)
f.deliver([]uint64{req.sectionIdx}, []BitVector{fetcherTestVector(req.bitIdx, req.sectionIdx)})
f.deliver([]uint64{req.sectionIdx}, [][]byte{fetcherTestVector(req.bitIdx, req.sectionIdx)})
}
}()
}

42
core/bloombits/matcher.go
View file

@ -20,6 +20,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/bitutil"
"github.com/ethereum/go-ethereum/core/types"
)
@ -36,7 +37,7 @@ type fetcher struct {
}
type req struct {
data BitVector
data []byte
queued bool
fetched chan struct{}
}
@ -50,8 +51,8 @@ type distReq struct {
// in the same order through the returned channel. Multiple fetch instances of the same fetcher are allowed
// to run in parallel, in case the same bit index appears multiple times in the filter structure. Each section
// is requested only once, requests are sent to the request distributor (part of Matcher) through distChn.
func (f *fetcher) fetch(sectionChn chan uint64, distChn chan distReq, stop chan struct{}, wg *sync.WaitGroup) chan BitVector {
dataChn := make(chan BitVector, channelCap)
func (f *fetcher) fetch(sectionChn chan uint64, distChn chan distReq, stop chan struct{}, wg *sync.WaitGroup) chan []byte {
dataChn := make(chan []byte, channelCap)
returnChn := make(chan uint64, channelCap)
wg.Add(2)
@ -137,7 +138,7 @@ func (f *fetcher) fetch(sectionChn chan uint64, distChn chan distReq, stop chan
// deliver is called by the request distributor when a reply to a request has
// arrived
func (f *fetcher) deliver(sectionIdxList []uint64, data []BitVector) {
func (f *fetcher) deliver(sectionIdxList []uint64, data [][]byte) {
f.reqLock.Lock()
defer f.reqLock.Unlock()
@ -213,7 +214,7 @@ loop:
// match creates a daisy-chain of sub-matchers, one for the address set and one for each topic set, each
// sub-matcher receiving a section only if the previous ones have all found a potential match in one of
// the blocks of the section, then binary and-ing its own matches and forwaring the result to the next one
func (m *Matcher) match(sectionChn chan uint64, stop chan struct{}) (chan uint64, chan BitVector) {
func (m *Matcher) match(sectionChn chan uint64, stop chan struct{}) (chan uint64, chan []byte) {
subIdx := m.topics
if len(m.addresses) > 0 {
subIdx = append([][]types.BloomIndexList{m.addresses}, subIdx...)
@ -223,7 +224,7 @@ func (m *Matcher) match(sectionChn chan uint64, stop chan struct{}) (chan uint64
m.distributeRequests(stop)
s := sectionChn
var bv chan BitVector
var bv chan []byte
for _, idx := range subIdx {
s, bv = m.subMatch(s, bv, idx, stop)
}
@ -246,10 +247,10 @@ func (m *Matcher) newFetcher(idx uint) {
// binary and-s the result to the daisy-chain input (sectionChn/andVectorChn) and forwards it to the daisy-chain output.
// The matches of each address/topic are calculated by fetching the given sections of the three bloom bit indexes belonging to
// that address/topic, and binary and-ing those vectors together.
func (m *Matcher) subMatch(sectionChn chan uint64, andVectorChn chan BitVector, idxs []types.BloomIndexList, stop chan struct{}) (chan uint64, chan BitVector) {
func (m *Matcher) subMatch(sectionChn chan uint64, andVectorChn chan []byte, idxs []types.BloomIndexList, stop chan struct{}) (chan uint64, chan []byte) {
// set up fetchers
fetchIdx := make([][3]chan uint64, len(idxs))
fetchData := make([][3]chan BitVector, len(idxs))
fetchData := make([][3]chan []byte, len(idxs))
for i, idx := range idxs {
for j, ii := range idx {
fetchIdx[i][j] = make(chan uint64, channelCap)
@ -259,7 +260,7 @@ func (m *Matcher) subMatch(sectionChn chan uint64, andVectorChn chan BitVector,
processChn := make(chan uint64, channelCap)
resIdxChn := make(chan uint64, channelCap)
resDataChn := make(chan BitVector, channelCap)
resDataChn := make(chan []byte, channelCap)
m.wg.Add(2)
// goroutine for starting retrievals
@ -314,41 +315,42 @@ func (m *Matcher) subMatch(sectionChn chan uint64, andVectorChn chan BitVector,
return
}
var orVector BitVector
var orVector []byte
for _, ff := range fetchData {
var andVector BitVector
var andVector []byte
for _, f := range ff {
var data BitVector
var data []byte
select {
case <-stop:
return
case data = <-f:
}
if andVector == nil {
andVector = bvCopy(data, int(m.sectionSize))
andVector = make([]byte, int(m.sectionSize/8))
copy(andVector, data)
} else {
bvAnd(andVector, data)
bitutil.ANDBytes(andVector, andVector, data)
}
}
if orVector == nil {
orVector = andVector
} else {
bvOr(orVector, andVector)
bitutil.ORBytes(orVector, orVector, andVector)
}
}
if orVector == nil {
orVector = bvZero(int(m.sectionSize))
orVector = make([]byte, int(m.sectionSize/8))
}
if andVectorChn != nil {
select {
case <-stop:
return
case andVector := <-andVectorChn:
bvAnd(orVector, andVector)
bitutil.ANDBytes(orVector, orVector, andVector)
}
}
if bvIsNonZero(orVector) {
if bitutil.TestBytes(orVector) {
select {
case <-stop:
return
@ -411,7 +413,7 @@ func (m *Matcher) GetMatches(start, end uint64, stop chan struct{}) chan uint64
if !ok {
return
}
var match BitVector
var match []byte
select {
case <-stop:
return
@ -570,6 +572,6 @@ func (m *Matcher) NextRequest(stop chan struct{}) (bitIdx uint, sectionIdxList [
// Deliver delivers a bit vector to the appropriate fetcher.
// It is possible to deliver data even after GetMatches has been stopped. Once a vector has been
// requested, the next call to GetMatches will keep waiting for delivery.
func (m *Matcher) Deliver(bitIdx uint, sectionIdxList []uint64, data []BitVector) {
func (m *Matcher) Deliver(bitIdx uint, sectionIdxList []uint64, data [][]byte) {
m.fetchers[bitIdx].deliver(sectionIdxList, data)
}

6
core/bloombits/matcher_test.go
View file

@ -25,8 +25,8 @@ import (
const testSectionSize = 4096
func matcherTestVector(b uint, s uint64) BitVector {
r := make(BitVector, testSectionSize/8)
func matcherTestVector(b uint, s uint64) []byte {
r := make([]byte, testSectionSize/8)
for i, _ := range r {
var bb byte
for bit := 0; bit < 8; bit++ {
@ -82,7 +82,7 @@ func testServeMatcher(m *Matcher, stop chan struct{}, cnt *uint32) {
if s == nil {
return
}
res := make([]BitVector, len(s))
res := make([][]byte, len(s))
for i, ss := range s {
res[i] = matcherTestVector(b, ss)
atomic.AddUint32(cnt, 1)

127
core/bloombits/utils.go
View file

@ -19,129 +19,6 @@ import (
"github.com/ethereum/go-ethereum/core/types"
)
type (
BitVector []byte
CompVector []byte
)
// bvAnd binary ANDs b to a
func bvAnd(a, b BitVector) {
for i, bb := range b {
a[i] &= bb
}
}
// bvOr binary ORs b to a
func bvOr(a, b BitVector) {
for i, bb := range b {
a[i] |= bb
}
}
// bvZero returns an all-zero bit vector
func bvZero(sectionSize int) BitVector {
return make(BitVector, sectionSize/8)
}
// bvCopy creates a copy of the given bit vector
// If the source vector is nil, returns an all-zero bit vector
func bvCopy(a BitVector, sectionSize int) BitVector {
c := make(BitVector, sectionSize/8)
copy(c, a)
return c
}
// bvIsNonZero returns true if the bit vector has at least one "1" bit
func bvIsNonZero(a BitVector) bool {
for _, b := range a {
if b != 0 {
return true
}
}
return false
}
// CompressBloomBits compresses a bit vector for storage/network transfer purposes
func CompressBloomBits(bits BitVector, sectionSize int) CompVector {
if len(bits) != sectionSize/8 {
panic(nil)
}
c := compressBits(bits)
if len(c) >= sectionSize/8 {
// make a copy so that output is always detached from input
return CompVector(bvCopy(bits, sectionSize))
}
return CompVector(c)
}
func compressBits(bits []byte) []byte {
l := len(bits)
ll := l / 8
if ll == 0 {
ll = 1
}
b := make([]byte, ll)
c := make([]byte, l)
cl := 0
for i, v := range bits {
if v != 0 {
c[cl] = v
cl++
b[i/8] |= 1 << byte(7-i%8)
}
}
if cl == 0 {
return nil
}
if ll > 1 {
b = compressBits(b)
}
return append(b, c[0:cl]...)
}
// DeompressBloomBits decompresses a bit vector
func DecompressBloomBits(bits CompVector, sectionSize int) BitVector {
if len(bits) == sectionSize/8 {
// make a copy so that output is always detached from input
return bvCopy(BitVector(bits), sectionSize)
}
dc, ofs := decompressBits(bits, sectionSize/8)
if ofs != len(bits) {
panic(nil)
}
return dc
}
func decompressBits(bits []byte, targetLen int) ([]byte, int) {
lb := len(bits)
dc := make([]byte, targetLen)
if lb == 0 {
return dc, 0
}
l := targetLen / 8
var (
b []byte
ofs int
)
if l <= 1 {
b = bits[0:1]
ofs = 1
} else {
b, ofs = decompressBits(bits, l)
}
for i, _ := range dc {
if b[i/8]&(1<<byte(7-i%8)) != 0 {
if ofs == lb {
panic(nil)
}
dc[i] = bits[ofs]
ofs++
}
}
return dc, ofs
}
const BloomLength = 2048
// BloomBitsCreator takes SectionSize number of header bloom filters and calculates the bloomBits vectors of the section
@ -177,10 +54,10 @@ func (b *BloomBitsCreator) AddHeaderBloom(bloom types.Bloom) {
}
// GetBitVector returns the bit vector belonging to the given bit index after header blooms have been added
func (b *BloomBitsCreator) GetBitVector(idx uint) BitVector {
func (b *BloomBitsCreator) GetBitVector(idx uint) []byte {
if b.bitIdx != b.sectionSize {
panic("not enough header blooms added")
}
return BitVector(b.blooms[idx][:])
return b.blooms[idx][:]
}

7
core/database_util.go
View file

@ -25,7 +25,6 @@ import (
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
@ -633,17 +632,17 @@ func FindCommonAncestor(db ethdb.Database, a, b *types.Header) *types.Header {
}
// GetBloomBits reads the compressed bloomBits vector belonging to the given section and bit index from the db
func GetBloomBits(db ethdb.Database, bitIdx, sectionIdx uint64) (bloombits.CompVector, error) {
func GetBloomBits(db ethdb.Database, bitIdx, sectionIdx uint64) ([]byte, error) {
var encKey [10]byte
binary.BigEndian.PutUint16(encKey[0:2], uint16(bitIdx))
binary.BigEndian.PutUint64(encKey[2:10], sectionIdx)
key := append(bloomBitsPrefix, encKey[:]...)
bloomBits, err := db.Get(key)
return bloombits.CompVector(bloomBits), err
return bloomBits, err
}
// StoreBloomBits writes the compressed bloomBits vector belonging to the given section and bit index to the db
func StoreBloomBits(db ethdb.Database, bitIdx, sectionIdx uint64, bloomBits bloombits.CompVector) {
func StoreBloomBits(db ethdb.Database, bitIdx, sectionIdx uint64, bloomBits []byte) {
var encKey [10]byte
binary.BigEndian.PutUint16(encKey[0:2], uint16(bitIdx))
binary.BigEndian.PutUint64(encKey[2:10], sectionIdx)

5
eth/api_backend.go
View file

@ -24,7 +24,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
@ -202,8 +201,8 @@ func (b *EthApiBackend) AccountManager() *accounts.Manager {
return b.eth.AccountManager()
}
func (b *EthApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([]bloombits.CompVector, error) {
results := make([]bloombits.CompVector, len(sectionIdxList))
func (b *EthApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) {
results := make([][]byte, len(sectionIdxList))
var err error
for i, idx := range sectionIdxList {
results[i], err = core.GetBloomBits(b.eth.chainDb, bitIdx, idx)

4
eth/db_upgrade.go
View file

@ -23,6 +23,7 @@ import (
"math/big"
"time"
"github.com/ethereum/go-ethereum/common/bitutil"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types"
@ -281,7 +282,8 @@ func (b *BloomBitsProcessorBackend) Process(sectionIdx uint64) bool {
}
for i := 0; i < bloombits.BloomLength; i++ {
core.StoreBloomBits(b.db, uint64(i), sectionIdx, bloombits.CompressBloomBits(bc.GetBitVector(uint(i)), bloomBitsSection))
compVector := bitutil.CompressBytes(bc.GetBitVector(uint(i)))
core.StoreBloomBits(b.db, uint64(i), sectionIdx, compVector)
}
return true

11
eth/filters/filter.go
View file

@ -22,6 +22,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/bitutil"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types"
@ -35,7 +36,7 @@ type Backend interface {
EventMux() *event.TypeMux
HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error)
GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([]bloombits.CompVector, error)
GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error)
}
// Filter can be used to retrieve and filter logs.
@ -141,9 +142,13 @@ func (f *Filter) serveMatcher(ctx context.Context, stop chan struct{}) chan erro
errChn <- err
return
}
decomp := make([]bloombits.BitVector, len(data))
decomp := make([][]byte, len(data))
for i, d := range data {
decomp[i] = bloombits.DecompressBloomBits(bloombits.CompVector(d), int(f.bloomBitsSection))
var err error
if decomp[i], err = bitutil.DecompressBytes(d, int(f.bloomBitsSection)); err != nil {
errChn <- err
return
}
}
f.matcher.Deliver(b, s, decomp)
}

5
eth/filters/filter_system_test.go
View file

@ -25,7 +25,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
@ -64,8 +63,8 @@ func (b *testBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (t
return core.GetBlockReceipts(b.db, blockHash, num), nil
}
func (b *testBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([]bloombits.CompVector, error) {
results := make([]bloombits.CompVector, len(sectionIdxList))
func (b *testBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) {
results := make([][]byte, len(sectionIdxList))
var err error
for i, idx := range sectionIdxList {
results[i], err = core.GetBloomBits(b.db, bitIdx, idx)

3
les/api_backend.go
View file

@ -24,7 +24,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/math"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/downloader"
@ -157,6 +156,6 @@ func (b *LesApiBackend) AccountManager() *accounts.Manager {
return b.eth.accountManager
}
func (b *LesApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([]bloombits.CompVector, error) {
func (b *LesApiBackend) GetBloomBits(ctx context.Context, bitIdx uint64, sectionIdxList []uint64) ([][]byte, error) {
return nil, nil // implemented in a subsequent PR
}