From 3dcccfb27c6c4aaba1f4862ceba189ebe380e3bc Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Tue, 9 May 2017 00:38:37 +0200 Subject: [PATCH] core/bloombits: removed bit vector aliases, using bitutil functions --- core/bloombits/fetcher_test.go | 6 +- core/bloombits/matcher.go | 42 +++++----- core/bloombits/matcher_test.go | 6 +- core/bloombits/utils.go | 127 +----------------------------- core/database_util.go | 7 +- eth/api_backend.go | 5 +- eth/db_upgrade.go | 4 +- eth/filters/filter.go | 11 ++- eth/filters/filter_system_test.go | 5 +- les/api_backend.go | 3 +- 10 files changed, 49 insertions(+), 167 deletions(-) diff --git a/core/bloombits/fetcher_test.go b/core/bloombits/fetcher_test.go index 3f436dfaa3..97425df532 100644 --- a/core/bloombits/fetcher_test.go +++ b/core/bloombits/fetcher_test.go @@ -27,8 +27,8 @@ import ( const testFetcherReqCnt = 50000 -func fetcherTestVector(b uint, s uint64) BitVector { - r := make(BitVector, 10) +func fetcherTestVector(b uint, s uint64) []byte { + r := make([]byte, 10) binary.BigEndian.PutUint16(r[0:2], uint16(b)) binary.BigEndian.PutUint64(r[2:10], s) return r @@ -59,7 +59,7 @@ func testFetcher(t *testing.T, cnt int) { } time.Sleep(time.Duration(rand.Intn(1000000))) atomic.AddUint32(&reqCnt, 1) - f.deliver([]uint64{req.sectionIdx}, []BitVector{fetcherTestVector(req.bitIdx, req.sectionIdx)}) + f.deliver([]uint64{req.sectionIdx}, [][]byte{fetcherTestVector(req.bitIdx, req.sectionIdx)}) } }() } diff --git a/core/bloombits/matcher.go b/core/bloombits/matcher.go index abd5bd734b..67970a9df8 100644 --- a/core/bloombits/matcher.go +++ b/core/bloombits/matcher.go @@ -20,6 +20,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/bitutil" "github.com/ethereum/go-ethereum/core/types" ) @@ -36,7 +37,7 @@ type fetcher struct { } type req struct { - data BitVector + data []byte queued bool fetched chan struct{} } @@ -50,8 +51,8 @@ type distReq struct { // in the same order through the returned channel. Multiple fetch instances of the same fetcher are allowed // to run in parallel, in case the same bit index appears multiple times in the filter structure. Each section // is requested only once, requests are sent to the request distributor (part of Matcher) through distChn. -func (f *fetcher) fetch(sectionChn chan uint64, distChn chan distReq, stop chan struct{}, wg *sync.WaitGroup) chan BitVector { - dataChn := make(chan BitVector, channelCap) +func (f *fetcher) fetch(sectionChn chan uint64, distChn chan distReq, stop chan struct{}, wg *sync.WaitGroup) chan []byte { + dataChn := make(chan []byte, channelCap) returnChn := make(chan uint64, channelCap) wg.Add(2) @@ -137,7 +138,7 @@ func (f *fetcher) fetch(sectionChn chan uint64, distChn chan distReq, stop chan // deliver is called by the request distributor when a reply to a request has // arrived -func (f *fetcher) deliver(sectionIdxList []uint64, data []BitVector) { +func (f *fetcher) deliver(sectionIdxList []uint64, data [][]byte) { f.reqLock.Lock() defer f.reqLock.Unlock() @@ -213,7 +214,7 @@ loop: // match creates a daisy-chain of sub-matchers, one for the address set and one for each topic set, each // sub-matcher receiving a section only if the previous ones have all found a potential match in one of // the blocks of the section, then binary and-ing its own matches and forwaring the result to the next one -func (m *Matcher) match(sectionChn chan uint64, stop chan struct{}) (chan uint64, chan BitVector) { +func (m *Matcher) match(sectionChn chan uint64, stop chan struct{}) (chan uint64, chan []byte) { subIdx := m.topics if len(m.addresses) > 0 { subIdx = append([][]types.BloomIndexList{m.addresses}, subIdx...) @@ -223,7 +224,7 @@ func (m *Matcher) match(sectionChn chan uint64, stop chan struct{}) (chan uint64 m.distributeRequests(stop) s := sectionChn - var bv chan BitVector + var bv chan []byte for _, idx := range subIdx { s, bv = m.subMatch(s, bv, idx, stop) } @@ -246,10 +247,10 @@ func (m *Matcher) newFetcher(idx uint) { // binary and-s the result to the daisy-chain input (sectionChn/andVectorChn) and forwards it to the daisy-chain output. // The matches of each address/topic are calculated by fetching the given sections of the three bloom bit indexes belonging to // that address/topic, and binary and-ing those vectors together. -func (m *Matcher) subMatch(sectionChn chan uint64, andVectorChn chan BitVector, idxs []types.BloomIndexList, stop chan struct{}) (chan uint64, chan BitVector) { +func (m *Matcher) subMatch(sectionChn chan uint64, andVectorChn chan []byte, idxs []types.BloomIndexList, stop chan struct{}) (chan uint64, chan []byte) { // set up fetchers fetchIdx := make([][3]chan uint64, len(idxs)) - fetchData := make([][3]chan BitVector, len(idxs)) + fetchData := make([][3]chan []byte, len(idxs)) for i, idx := range idxs { for j, ii := range idx { fetchIdx[i][j] = make(chan uint64, channelCap) @@ -259,7 +260,7 @@ func (m *Matcher) subMatch(sectionChn chan uint64, andVectorChn chan BitVector, processChn := make(chan uint64, channelCap) resIdxChn := make(chan uint64, channelCap) - resDataChn := make(chan BitVector, channelCap) + resDataChn := make(chan []byte, channelCap) m.wg.Add(2) // goroutine for starting retrievals @@ -314,41 +315,42 @@ func (m *Matcher) subMatch(sectionChn chan uint64, andVectorChn chan BitVector, return } - var orVector BitVector + var orVector []byte for _, ff := range fetchData { - var andVector BitVector + var andVector []byte for _, f := range ff { - var data BitVector + var data []byte select { case <-stop: return case data = <-f: } if andVector == nil { - andVector = bvCopy(data, int(m.sectionSize)) + andVector = make([]byte, int(m.sectionSize/8)) + copy(andVector, data) } else { - bvAnd(andVector, data) + bitutil.ANDBytes(andVector, andVector, data) } } if orVector == nil { orVector = andVector } else { - bvOr(orVector, andVector) + bitutil.ORBytes(orVector, orVector, andVector) } } if orVector == nil { - orVector = bvZero(int(m.sectionSize)) + orVector = make([]byte, int(m.sectionSize/8)) } if andVectorChn != nil { select { case <-stop: return case andVector := <-andVectorChn: - bvAnd(orVector, andVector) + bitutil.ANDBytes(orVector, orVector, andVector) } } - if bvIsNonZero(orVector) { + if bitutil.TestBytes(orVector) { select { case <-stop: return @@ -411,7 +413,7 @@ func (m *Matcher) GetMatches(start, end uint64, stop chan struct{}) chan uint64 if !ok { return } - var match BitVector + var match []byte select { case <-stop: return @@ -570,6 +572,6 @@ func (m *Matcher) NextRequest(stop chan struct{}) (bitIdx uint, sectionIdxList [ // Deliver delivers a bit vector to the appropriate fetcher. // It is possible to deliver data even after GetMatches has been stopped. Once a vector has been // requested, the next call to GetMatches will keep waiting for delivery. -func (m *Matcher) Deliver(bitIdx uint, sectionIdxList []uint64, data []BitVector) { +func (m *Matcher) Deliver(bitIdx uint, sectionIdxList []uint64, data [][]byte) { m.fetchers[bitIdx].deliver(sectionIdxList, data) } diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go index 21d617b840..ac32b2de85 100644 --- a/core/bloombits/matcher_test.go +++ b/core/bloombits/matcher_test.go @@ -25,8 +25,8 @@ import ( const testSectionSize = 4096 -func matcherTestVector(b uint, s uint64) BitVector { - r := make(BitVector, testSectionSize/8) +func matcherTestVector(b uint, s uint64) []byte { + r := make([]byte, testSectionSize/8) for i, _ := range r { var bb byte for bit := 0; bit < 8; bit++ { @@ -82,7 +82,7 @@ func testServeMatcher(m *Matcher, stop chan struct{}, cnt *uint32) { if s == nil { return } - res := make([]BitVector, len(s)) + res := make([][]byte, len(s)) for i, ss := range s { res[i] = matcherTestVector(b, ss) atomic.AddUint32(cnt, 1) diff --git a/core/bloombits/utils.go b/core/bloombits/utils.go index 166a7d6039..e76d51f24a 100644 --- a/core/bloombits/utils.go +++ b/core/bloombits/utils.go @@ -19,129 +19,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -type ( - BitVector []byte - CompVector []byte -) - -// bvAnd binary ANDs b to a -func bvAnd(a, b BitVector) { - for i, bb := range b { - a[i] &= bb - } -} - -// bvOr binary ORs b to a -func bvOr(a, b BitVector) { - for i, bb := range b { - a[i] |= bb - } -} - -// bvZero returns an all-zero bit vector -func bvZero(sectionSize int) BitVector { - return make(BitVector, sectionSize/8) -} - -// bvCopy creates a copy of the given bit vector -// If the source vector is nil, returns an all-zero bit vector -func bvCopy(a BitVector, sectionSize int) BitVector { - c := make(BitVector, sectionSize/8) - copy(c, a) - return c -} - -// bvIsNonZero returns true if the bit vector has at least one "1" bit -func bvIsNonZero(a BitVector) bool { - for _, b := range a { - if b != 0 { - return true - } - } - return false -} - -// CompressBloomBits compresses a bit vector for storage/network transfer purposes -func CompressBloomBits(bits BitVector, sectionSize int) CompVector { - if len(bits) != sectionSize/8 { - panic(nil) - } - c := compressBits(bits) - if len(c) >= sectionSize/8 { - // make a copy so that output is always detached from input - return CompVector(bvCopy(bits, sectionSize)) - } - return CompVector(c) -} - -func compressBits(bits []byte) []byte { - l := len(bits) - ll := l / 8 - if ll == 0 { - ll = 1 - } - b := make([]byte, ll) - c := make([]byte, l) - cl := 0 - for i, v := range bits { - if v != 0 { - c[cl] = v - cl++ - b[i/8] |= 1 << byte(7-i%8) - } - } - if cl == 0 { - return nil - } - if ll > 1 { - b = compressBits(b) - } - return append(b, c[0:cl]...) -} - -// DeompressBloomBits decompresses a bit vector -func DecompressBloomBits(bits CompVector, sectionSize int) BitVector { - if len(bits) == sectionSize/8 { - // make a copy so that output is always detached from input - return bvCopy(BitVector(bits), sectionSize) - } - dc, ofs := decompressBits(bits, sectionSize/8) - if ofs != len(bits) { - panic(nil) - } - return dc -} - -func decompressBits(bits []byte, targetLen int) ([]byte, int) { - lb := len(bits) - dc := make([]byte, targetLen) - if lb == 0 { - return dc, 0 - } - - l := targetLen / 8 - var ( - b []byte - ofs int - ) - if l <= 1 { - b = bits[0:1] - ofs = 1 - } else { - b, ofs = decompressBits(bits, l) - } - for i, _ := range dc { - if b[i/8]&(1<