swarm/storage: use dummy memstore

This commit is contained in:
Anton Evangelatov 2018-03-30 18:55:56 +02:00
parent 3ec0007ce4
commit 4809d94999
3 changed files with 308 additions and 309 deletions

View file

@ -78,9 +78,9 @@ func NewTestLocalStoreForAddr(path string, basekey []byte) (*LocalStore, error)
return localStore, nil
}
func (self *LocalStore) CacheCounter() uint64 {
return uint64(self.memStore.Counter())
}
//func (self *LocalStore) CacheCounter() uint64 {
//return uint64(self.memStore.Counter())
//}
// LocalStore is itself a chunk store
// unsafe, in that the data is not integrity checked

View file

@ -19,11 +19,10 @@
package storage
import (
"bytes"
"fmt"
"sync"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
)
@ -40,14 +39,14 @@ const (
defaultCacheCapacity = 5000
)
type MemStore struct {
memtree *memTree
entryCnt, capacity uint // stored entries
accessCnt uint64 // access counter; oldest is thrown away when full
dbAccessCnt uint64
ldbStore *LDBStore
lock sync.Mutex
}
//type MemStore struct {
//memtree *memTree
//entryCnt, capacity uint // stored entries
//accessCnt uint64 // access counter; oldest is thrown away when full
//dbAccessCnt uint64
//ldbStore *LDBStore
//lock sync.Mutex
//}
/*
a hash prefix subtree containing subtrees or one storage entry (but never both)
@ -62,312 +61,312 @@ a hash prefix subtree containing subtrees or one storage entry (but never both)
(access[] is a binary tree inside the multi-bit leveled hash tree)
*/
//func NewMemStore(d *LDBStore, capacity uint) (m *MemStore) {
//m = &MemStore{}
//m.memtree = newMemTree(memTreeFLW, nil, 0)
//m.ldbStore = d
//m.setCapacity(capacity)
//return
//}
//type memTree struct {
//subtree []*memTree
//parent *memTree
//parentIdx uint
//bits uint // log2(subtree count)
//width uint // subtree count
//entry *Chunk // if subtrees are present, entry should be nil
//lastDBaccess uint64
//access []uint64
//}
//func newMemTree(b uint, parent *memTree, pidx uint) (node *memTree) {
//node = new(memTree)
//node.bits = b
//node.width = 1 << b
//node.subtree = make([]*memTree, node.width)
//node.access = make([]uint64, node.width-1)
//node.parent = parent
//node.parentIdx = pidx
//if parent != nil {
//parent.subtree[pidx] = node
//}
//return node
//}
//func (node *memTree) updateAccess(a uint64) {
//aidx := uint(0)
//var aa uint64
//oa := node.access[0]
//for node.access[aidx] == oa {
//node.access[aidx] = a
//if aidx > 0 {
//aa = node.access[((aidx-1)^1)+1]
//aidx = (aidx - 1) >> 1
//} else {
//pidx := node.parentIdx
//node = node.parent
//if node == nil {
//return
//}
//nn := node.subtree[pidx^1]
//if nn != nil {
//aa = nn.access[0]
//} else {
//aa = 0
//}
//aidx = (node.width + pidx - 2) >> 1
//}
//if (aa != 0) && (aa < a) {
//a = aa
//}
//}
//}
//func (s *MemStore) setCapacity(c uint) {
//s.lock.Lock()
//defer s.lock.Unlock()
//for c < s.entryCnt {
//s.removeOldest()
//}
//s.capacity = c
//}
//func (s *MemStore) Counter() uint {
//return s.entryCnt
//}
//// entry (not its copy) is going to be in MemStore
//func (s *MemStore) Put(entry *Chunk) {
//log.Trace("memstore.put", "key", entry.Key)
//if s.capacity == 0 {
//return
//}
//s.lock.Lock()
//defer s.lock.Unlock()
//if s.entryCnt >= s.capacity {
//s.removeOldest()
//}
//s.accessCnt++
//memstorePutCounter.Inc(1)
//node := s.memtree
//bitpos := uint(0)
//for node.entry == nil {
//l := entry.Key.bits(bitpos, node.bits)
//st := node.subtree[l]
//if st == nil {
//st = newMemTree(memTreeLW, node, l)
//bitpos += node.bits
//node = st
//break
//}
//bitpos += node.bits
//node = st
//}
//if node.entry != nil {
//if node.entry.Key.isEqual(entry.Key) {
//node.updateAccess(s.accessCnt)
//if entry.SData == nil {
//entry.Size = node.entry.Size
//entry.SData = node.entry.SData
//}
//if entry.ReqC == nil {
//entry.ReqC = node.entry.ReqC
//}
//entry.C = node.entry.C
//node.entry = entry
//return
//}
//for node.entry != nil {
//l := node.entry.Key.bits(bitpos, node.bits)
//st := node.subtree[l]
//if st == nil {
//st = newMemTree(memTreeLW, node, l)
//}
//st.entry = node.entry
//node.entry = nil
//st.updateAccess(node.access[0])
//l = entry.Key.bits(bitpos, node.bits)
//st = node.subtree[l]
//if st == nil {
//st = newMemTree(memTreeLW, node, l)
//}
//bitpos += node.bits
//node = st
//}
//}
//node.entry = entry
//node.lastDBaccess = s.dbAccessCnt
//node.updateAccess(s.accessCnt)
//s.entryCnt++
//}
//func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) {
//log.Trace("memstore.get", "key", hash)
//s.lock.Lock()
//defer s.lock.Unlock()
//node := s.memtree
//bitpos := uint(0)
//for node.entry == nil {
//l := hash.bits(bitpos, node.bits)
//st := node.subtree[l]
//if st == nil {
//log.Trace("memstore.get ErrChunkNotFound", "key", hash)
//return nil, ErrChunkNotFound
//}
//bitpos += node.bits
//node = st
//}
//if node.entry.Key.isEqual(hash) {
//s.accessCnt++
//node.updateAccess(s.accessCnt)
//chunk = node.entry
//if s.dbAccessCnt-node.lastDBaccess > dbForceUpdateAccessCnt {
//s.dbAccessCnt++
//node.lastDBaccess = s.dbAccessCnt
//if s.ldbStore != nil {
//s.ldbStore.updateAccessCnt(hash)
//}
//}
//} else {
//err = ErrChunkNotFound
//}
//log.Trace("memstore.get return", "key", hash, "chunk", chunk, "err", err)
//return
//}
//func (s *MemStore) removeOldest() {
//defer metrics.GetOrRegisterResettingTimer("memstore.purge", metrics.DefaultRegistry).UpdateSince(time.Now())
//node := s.memtree
//log.Warn("purge memstore")
//for node.entry == nil {
//aidx := uint(0)
//av := node.access[aidx]
//for aidx < node.width/2-1 {
//if av == node.access[aidx*2+1] {
//node.access[aidx] = node.access[aidx*2+2]
//aidx = aidx*2 + 1
//} else if av == node.access[aidx*2+2] {
//node.access[aidx] = node.access[aidx*2+1]
//aidx = aidx*2 + 2
//} else {
//panic(nil)
//}
//}
//pidx := aidx*2 + 2 - node.width
//if (node.subtree[pidx] != nil) && (av == node.subtree[pidx].access[0]) {
//if node.subtree[pidx+1] != nil {
//node.access[aidx] = node.subtree[pidx+1].access[0]
//} else {
//node.access[aidx] = 0
//}
//} else if (node.subtree[pidx+1] != nil) && (av == node.subtree[pidx+1].access[0]) {
//if node.subtree[pidx] != nil {
//node.access[aidx] = node.subtree[pidx].access[0]
//} else {
//node.access[aidx] = 0
//}
//pidx++
//} else {
//panic(nil)
//}
////fmt.Println(pidx)
//node = node.subtree[pidx]
//}
//if node.entry.ReqC == nil {
//log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log()))
//<-node.entry.dbStoredC
//log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log()))
//memstoreRemoveCounter.Inc(1)
//node.entry = nil
//s.entryCnt--
//} else {
//return
//}
//node.access[0] = 0
////---
//aidx := uint(0)
//for {
//aa := node.access[aidx]
//if aidx > 0 {
//aidx = (aidx - 1) >> 1
//} else {
//pidx := node.parentIdx
//node = node.parent
//if node == nil {
//return
//}
//aidx = (node.width + pidx - 2) >> 1
//}
//if (aa != 0) && ((aa < node.access[aidx]) || (node.access[aidx] == 0)) {
//node.access[aidx] = aa
//}
//}
//}
type MemStore struct {
m map[string]*Chunk
mu sync.RWMutex
}
func NewMemStore(d *LDBStore, capacity uint) (m *MemStore) {
m = &MemStore{}
m.memtree = newMemTree(memTreeFLW, nil, 0)
m.ldbStore = d
m.setCapacity(capacity)
return
}
type memTree struct {
subtree []*memTree
parent *memTree
parentIdx uint
bits uint // log2(subtree count)
width uint // subtree count
entry *Chunk // if subtrees are present, entry should be nil
lastDBaccess uint64
access []uint64
}
func newMemTree(b uint, parent *memTree, pidx uint) (node *memTree) {
node = new(memTree)
node.bits = b
node.width = 1 << b
node.subtree = make([]*memTree, node.width)
node.access = make([]uint64, node.width-1)
node.parent = parent
node.parentIdx = pidx
if parent != nil {
parent.subtree[pidx] = node
}
return node
}
func (node *memTree) updateAccess(a uint64) {
aidx := uint(0)
var aa uint64
oa := node.access[0]
for node.access[aidx] == oa {
node.access[aidx] = a
if aidx > 0 {
aa = node.access[((aidx-1)^1)+1]
aidx = (aidx - 1) >> 1
} else {
pidx := node.parentIdx
node = node.parent
if node == nil {
return
}
nn := node.subtree[pidx^1]
if nn != nil {
aa = nn.access[0]
} else {
aa = 0
}
aidx = (node.width + pidx - 2) >> 1
}
if (aa != 0) && (aa < a) {
a = aa
}
return &MemStore{
m: make(map[string]*Chunk),
}
}
func (s *MemStore) setCapacity(c uint) {
s.lock.Lock()
defer s.lock.Unlock()
for c < s.entryCnt {
s.removeOldest()
func (m *MemStore) Get(key Key) (*Chunk, error) {
m.mu.RLock()
defer m.mu.RUnlock()
c, ok := m.m[string(key[:])]
if !ok {
return nil, ErrChunkNotFound
}
s.capacity = c
if !bytes.Equal(c.Key, key) {
panic(fmt.Errorf("MemStore.Get: chunk key %s != req key %s", c.Key.Hex(), key.Hex()))
}
return c, nil
}
func (s *MemStore) Counter() uint {
return s.entryCnt
func (m *MemStore) Put(c *Chunk) {
m.mu.Lock()
defer m.mu.Unlock()
m.m[string(c.Key[:])] = c
}
// entry (not its copy) is going to be in MemStore
func (s *MemStore) Put(entry *Chunk) {
log.Trace("memstore.put", "key", entry.Key)
if s.capacity == 0 {
return
}
func (m *MemStore) setCapacity(n int) {
s.lock.Lock()
defer s.lock.Unlock()
if s.entryCnt >= s.capacity {
s.removeOldest()
}
s.accessCnt++
memstorePutCounter.Inc(1)
node := s.memtree
bitpos := uint(0)
for node.entry == nil {
l := entry.Key.bits(bitpos, node.bits)
st := node.subtree[l]
if st == nil {
st = newMemTree(memTreeLW, node, l)
bitpos += node.bits
node = st
break
}
bitpos += node.bits
node = st
}
if node.entry != nil {
if node.entry.Key.isEqual(entry.Key) {
node.updateAccess(s.accessCnt)
if entry.SData == nil {
entry.Size = node.entry.Size
entry.SData = node.entry.SData
}
if entry.ReqC == nil {
entry.ReqC = node.entry.ReqC
}
entry.C = node.entry.C
node.entry = entry
return
}
for node.entry != nil {
l := node.entry.Key.bits(bitpos, node.bits)
st := node.subtree[l]
if st == nil {
st = newMemTree(memTreeLW, node, l)
}
st.entry = node.entry
node.entry = nil
st.updateAccess(node.access[0])
l = entry.Key.bits(bitpos, node.bits)
st = node.subtree[l]
if st == nil {
st = newMemTree(memTreeLW, node, l)
}
bitpos += node.bits
node = st
}
}
node.entry = entry
node.lastDBaccess = s.dbAccessCnt
node.updateAccess(s.accessCnt)
s.entryCnt++
}
func (s *MemStore) Get(hash Key) (chunk *Chunk, err error) {
log.Trace("memstore.get", "key", hash)
s.lock.Lock()
defer s.lock.Unlock()
node := s.memtree
bitpos := uint(0)
for node.entry == nil {
l := hash.bits(bitpos, node.bits)
st := node.subtree[l]
if st == nil {
log.Trace("memstore.get ErrChunkNotFound", "key", hash)
return nil, ErrChunkNotFound
}
bitpos += node.bits
node = st
}
if node.entry.Key.isEqual(hash) {
s.accessCnt++
node.updateAccess(s.accessCnt)
chunk = node.entry
if s.dbAccessCnt-node.lastDBaccess > dbForceUpdateAccessCnt {
s.dbAccessCnt++
node.lastDBaccess = s.dbAccessCnt
if s.ldbStore != nil {
s.ldbStore.updateAccessCnt(hash)
}
}
} else {
err = ErrChunkNotFound
}
log.Trace("memstore.get return", "key", hash, "chunk", chunk, "err", err)
return
}
func (s *MemStore) removeOldest() {
defer metrics.GetOrRegisterResettingTimer("memstore.purge", metrics.DefaultRegistry).UpdateSince(time.Now())
node := s.memtree
log.Warn("purge memstore")
for node.entry == nil {
aidx := uint(0)
av := node.access[aidx]
for aidx < node.width/2-1 {
if av == node.access[aidx*2+1] {
node.access[aidx] = node.access[aidx*2+2]
aidx = aidx*2 + 1
} else if av == node.access[aidx*2+2] {
node.access[aidx] = node.access[aidx*2+1]
aidx = aidx*2 + 2
} else {
panic(nil)
}
}
pidx := aidx*2 + 2 - node.width
if (node.subtree[pidx] != nil) && (av == node.subtree[pidx].access[0]) {
if node.subtree[pidx+1] != nil {
node.access[aidx] = node.subtree[pidx+1].access[0]
} else {
node.access[aidx] = 0
}
} else if (node.subtree[pidx+1] != nil) && (av == node.subtree[pidx+1].access[0]) {
if node.subtree[pidx] != nil {
node.access[aidx] = node.subtree[pidx].access[0]
} else {
node.access[aidx] = 0
}
pidx++
} else {
panic(nil)
}
//fmt.Println(pidx)
node = node.subtree[pidx]
}
if node.entry.ReqC == nil {
log.Trace(fmt.Sprintf("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log()))
<-node.entry.dbStoredC
log.Trace(fmt.Sprintf("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log()))
memstoreRemoveCounter.Inc(1)
node.entry = nil
s.entryCnt--
} else {
return
}
node.access[0] = 0
//---
aidx := uint(0)
for {
aa := node.access[aidx]
if aidx > 0 {
aidx = (aidx - 1) >> 1
} else {
pidx := node.parentIdx
node = node.parent
if node == nil {
return
}
aidx = (node.width + pidx - 2) >> 1
}
if (aa != 0) && ((aa < node.access[aidx]) || (node.access[aidx] == 0)) {
node.access[aidx] = aa
}
}
}
// type MemStore struct {
// m map[string]*Chunk
// mu sync.RWMutex
// }
// func NewMemStore(d *DbStore, capacity uint) (m *MemStore) {
// return &MemStore{
// m: make(map[string]*Chunk),
// }
// }
// func (m *MemStore) Get(key Key) (*Chunk, error) {
// m.mu.RLock()
// defer m.mu.RUnlock()
// c, ok := m.m[string(key[:])]
// if !ok {
// return nil, ErrNotFound
// }
// if !bytes.Equal(c.Key, key) {
// panic(fmt.Errorf("MemStore.Get: chunk key %s != req key %s", c.Key.Hex(), key.Hex()))
// }
// return c, nil
// }
// func (m *MemStore) Put(c *Chunk) {
// m.mu.Lock()
// defer m.mu.Unlock()
// m.m[string(c.Key[:])] = c
// }
// func (m *MemStore) setCapacity(n int) {
// }
// Close memstore
func (s *MemStore) Close() {}

View file

@ -394,7 +394,7 @@ func (self *Swarm) periodicallyUpdateGauges() {
}
func (self *Swarm) updateGauges() {
cacheSizeGauge.Update(int64(self.lstore.CacheCounter()))
//cacheSizeGauge.Update(int64(self.lstore.CacheCounter()))
uptimeGauge.Update(time.Since(startTime).Nanoseconds())
}