Added setCapacity, getEntryCnt for mem/dbStore

bzz/dbstore.go

@@ -13,10 +13,8 @@ import (
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
-const dbMaxEntries = 5000 // max number of stored (cached) blocks
+const gcArraySize = 10000
-
+const gcArrayFreeRatio = 0.1
-const gcArraySize = 500
-const gcArrayFreeRatio = 10
 
 // key prefixes for leveldb storage
 const kpIndex = 0
@@ -37,7 +35,7 @@ type dbStore struct {
 	db *ethdb.LDBDatabase
 
 	// this should be stored in db, accessed transactionally
-	entryCnt, accessCnt, dataIdx uint64
+	entryCnt, accessCnt, dataIdx, capacity uint64
 
 	gcPos, gcStartPos []byte
 	gcArray           []*gcItem
@@ -183,7 +181,7 @@ func gcListSelect(list []*gcItem, left int, right int, n int) int {
 	}
 }
 
-func (s *dbStore) collectGarbage() {
+func (s *dbStore) collectGarbage(ratio float32) {
 
 	it := s.db.NewIterator()
 	it.Seek(s.gcPos)
@@ -226,7 +224,7 @@ func (s *dbStore) collectGarbage() {
 		}
 	}
 
-	cutidx := gcListSelect(s.gcArray, 0, gcnt-1, gcnt/gcArrayFreeRatio)
+	cutidx := gcListSelect(s.gcArray, 0, gcnt-1, int(float32(gcnt)*ratio))
 	cutval := s.gcArray[cutidx].value
 
 	//fmt.Print(s.entryCnt, " ")
@@ -264,8 +262,8 @@ func (s *dbStore) Put(chunk *Chunk) {
 	data := encodeData(chunk)
 	//data := ethutil.Encode([]interface{}{entry})
 
-	if s.entryCnt >= dbMaxEntries {
+	if s.entryCnt >= s.capacity {
-		s.collectGarbage()
+		s.collectGarbage(gcArrayFreeRatio)
 	}
 
 	batch := new(leveldb.Batch)
@@ -346,6 +344,29 @@ func (s *dbStore) updateAccessCnt(key Key) {
 
 }
 
+func (s *dbStore) setCapacity(c uint64) {
+
+	s.capacity = c
+
+	if s.entryCnt > c {
+		var ratio float32
+		ratio = float32(0.99) - float32(c)/float32(s.entryCnt)
+		if ratio < 0 {
+			ratio = 0
+		}
+		for s.entryCnt > c {
+			s.collectGarbage(ratio)
+		}
+	}
+
+}
+
+func (s *dbStore) getEntryCnt() uint64 {
+
+	return s.entryCnt
+
+}
+
 func newDbStore(path string) (s *dbStore, err error) {
 
 	s = new(dbStore)
@@ -355,6 +376,8 @@ func newDbStore(path string) (s *dbStore, err error) {
 		return
 	}
 
+	s.setCapacity(5000)
+
 	s.gcStartPos = make([]byte, 1)
 	s.gcStartPos[0] = kpIndex
 	s.gcArray = make([]*gcItem, gcArraySize)

bzz/memstore.go

@@ -8,19 +8,18 @@ import (
 )
 
 const (
-	maxEntries             = 500 // max number of stored (cached) blocks
+	memTreeLW              = 2  // log2(subtree count) of the subtrees
-	memTreeLW              = 2   // log2(subtree count) of the subtrees
+	memTreeFLW             = 14 // log2(subtree count) of the root layer
-	memTreeFLW             = 14  // log2(subtree count) of the root layer
 	dbForceUpdateAccessCnt = 1000
 )
 
 type memStore struct {
-	memtree     *memTree
+	memtree            *memTree
-	entryCnt    uint   // stored entries
+	entryCnt, capacity uint   // stored entries
-	accessCnt   uint64 // access counter; oldest is thrown away when full
+	accessCnt          uint64 // access counter; oldest is thrown away when full
-	dbAccessCnt uint64
+	dbAccessCnt        uint64
-	dbStore     *dbStore
+	dbStore            *dbStore
-	lock        sync.Mutex
+	lock               sync.Mutex
 }
 
 /*
@@ -40,6 +39,7 @@ func newMemStore(d *dbStore) (m *memStore) {
 	m = &memStore{}
 	m.memtree = newMemTree(memTreeFLW, nil, 0)
 	m.dbStore = d
+	m.setCapacity(500)
 	return
 }
 
@@ -141,12 +141,34 @@ func (node *memTree) updateAccess(a uint64) {
 
 }
 
-func (s *memStore) Put(entry *Chunk) {
+func (s *memStore) setCapacity(c uint) {
 
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
-	if s.entryCnt >= maxEntries {
+	for c < s.entryCnt {
+		s.removeOldest()
+	}
+	s.capacity = c
+
+}
+
+func (s *memStore) getEntryCnt() uint {
+
+	return s.entryCnt
+
+}
+
+func (s *memStore) Put(entry *Chunk) {
+
+	if s.capacity == 0 {
+		return
+	}
+
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.entryCnt >= s.capacity {
 		s.removeOldest()
 	}