les: rework clientpool

les/clientpool.go

@@ -17,41 +17,58 @@
 package les
 
 import (
+	"encoding/binary"
 	"io"
 	"math"
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/mclock"
 	"github.com/ethereum/go-ethereum/common/prque"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/p2p/enode"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/hashicorp/golang-lru"
 )
 
 const (
-	negBalanceExpTC      = time.Hour        // time constant for exponentially reducing negative balance
+	negBalanceExpTC       = time.Minute      // time constant for exponentially reducing negative balance
-	fixedPointMultiplier = 0x1000000        // constant to convert logarithms to fixed point format
+	fixedPointMultiplier  = 0x1000000        // constant to convert logarithms to fixed point format
-	connectedBias        = time.Minute * 5  // this bias is applied in favor of already connected clients in order to avoid kicking them out very soon
+	lazyQueueRefresh      = time.Second * 10 // refresh period of the connected queue
-	lazyQueueRefresh     = time.Second * 10 // refresh period of the connected queue
+	persistCumTimeRefresh = time.Minute * 5  // refresh period of the cumulative running time persistence
-)
+	posBalanceCacheLimit  = 8192             // the maximum number of cached items in positive balance queue
+	negBalanceCacheLimit  = 8192             // the maximum number of cached items in negative balance queue
 
-var (
+	// freeConnectedBias is applied to already connected clients when new client
-	clientPoolDbKey    = []byte("clientPool")
+	// is "free client". So that already connected client won't be kicked out very
-	clientBalanceDbKey = []byte("clientPool-balance")
+	// soon.
+	freeConnectedBias = time.Minute * 5
+
+	// priorityConnectedBias is applied to already connected clients when new client
+	// is "priority client". The value is smaller than freeConnectedBias, so that
+	// we can ensure most of new priority client can be accepted when pool is full.
+	// But if the balance of priority client is very small, there is no reason for
+	// very high priority.
+	priorityConnectedBias = time.Minute
 )
 
 // clientPool implements a client database that assigns a priority to each client
 // based on a positive and negative balance. Positive balance is externally assigned
 // to prioritized clients and is decreased with connection time and processed
 // requests (unless the price factors are zero). If the positive balance is zero
-// then negative balance is accumulated. Balance tracking and priority calculation
+// then negative balance is accumulated.
-// for connected clients is done by balanceTracker. connectedQueue ensures that
+//
-// clients with the lowest positive or highest negative balance get evicted when
+// Balance tracking and priority calculation for connected clients is done by
-// the total capacity allowance is full and new clients with a better balance want
+// balanceTracker. connectedQueue ensures that clients with the lowest positive or
-// to connect. Already connected nodes receive a small bias in their favor in order
+// highest negative balance get evicted when the total capacity allowance is full
-// to avoid accepting and instantly kicking out clients.
+// and new clients with a better balance want to connect.
+//
+// Already connected nodes receive a small bias in their favor in order to avoid
+// accepting and instantly kicking out clients. In theory, we try to ensure that
+// each client can have several minutes of connection time.
+//
 // Balances of disconnected clients are stored in posBalanceQueue and negBalanceQueue
 // and are also saved in the database. Negative balance is transformed into a
 // logarithmic form with a constantly shifting linear offset in order to implement
@@ -59,25 +76,25 @@ var (
 // values when necessary. Positive balances are stored in the database as long as
 // they exist, posBalanceQueue only acts as a cache for recently accessed entries.
 type clientPool struct {
-	db         ethdb.Database
+	ndb        *nodeDB
 	lock       sync.Mutex
 	clock      mclock.Clock
-	stopCh     chan chan struct{}
+	stopCh     chan struct{}
 	closed     bool
 	removePeer func(enode.ID)
 
-	queueLimit, countLimit                          int
+	connectedMap   map[enode.ID]*clientInfo
-	freeClientCap, capacityLimit, connectedCapacity uint64
+	connectedQueue *prque.LazyQueue
 
-	connectedMap                     map[enode.ID]*clientInfo
+	posFactors, negFactors priceFactors
-	posBalanceMap                    map[enode.ID]*posBalance
+
-	negBalanceMap                    map[string]*negBalance
+	connLimit     int            // The maximum number of connections that clientpool can support
-	connectedQueue                   *prque.LazyQueue
+	capLimit      uint64         // The maximum cumulative capacity that clientpool can support
-	posBalanceQueue, negBalanceQueue *prque.Prque
+	connectedCap  uint64         // The sum of the capacity of the current clientpool connected
-	posFactors, negFactors           priceFactors
+	freeClientCap uint64         // The capacity value of each free client
-	posBalanceAccessCounter          int64
+	startTime     mclock.AbsTime // The timestamp at which the clientpool started running
-	startupTime                      mclock.AbsTime
+	startCumTime  int64          // The cumulative running time of clientpool at the start point.
-	logOffsetAtStartup               int64
+	disableBias   bool           // Disable connection bias(used in testing)
 }
 
 // clientPeer represents a client in the pool.
@@ -138,22 +155,28 @@ type priceFactors struct {
 }
 
 // newClientPool creates a new client pool
-func newClientPool(db ethdb.Database, freeClientCap uint64, queueLimit int, clock mclock.Clock, removePeer func(enode.ID)) *clientPool {
+func newClientPool(db ethdb.Database, freeClientCap uint64, clock mclock.Clock, removePeer func(enode.ID)) *clientPool {
+	ndb := newNodeDB(db, clock)
 	pool := &clientPool{
-		db:              db,
+		ndb:            ndb,
-		clock:           clock,
+		clock:          clock,
-		connectedMap:    make(map[enode.ID]*clientInfo),
+		connectedMap:   make(map[enode.ID]*clientInfo),
-		posBalanceMap:   make(map[enode.ID]*posBalance),
+		connectedQueue: prque.NewLazyQueue(connSetIndex, connPriority, connMaxPriority, clock, lazyQueueRefresh),
-		negBalanceMap:   make(map[string]*negBalance),
+		freeClientCap:  freeClientCap,
-		connectedQueue:  prque.NewLazyQueue(connSetIndex, connPriority, connMaxPriority, clock, lazyQueueRefresh),
+		removePeer:     removePeer,
-		negBalanceQueue: prque.New(negSetIndex),
+		startTime:      clock.Now(),
-		posBalanceQueue: prque.New(posSetIndex),
+		startCumTime:   ndb.getCumTime(),
-		freeClientCap:   freeClientCap,
+		stopCh:         make(chan struct{}),
-		queueLimit:      queueLimit,
+	}
-		removePeer:      removePeer,
+	// If the negative balance of free client is even lower than 1,
-		stopCh:          make(chan chan struct{}),
+	// delete this entry.
+	ndb.nbEvictCallBack = func(now mclock.AbsTime, b negBalance) bool {
+		balance := math.Exp(float64(b.logValue-pool.logOffset(now)) / fixedPointMultiplier)
+		if balance <= 1 {
+			return true
+		}
+		return false
 	}
-	pool.loadFromDb()
 	go func() {
 		for {
 			select {
@@ -161,8 +184,9 @@ func newClientPool(db ethdb.Database, freeClientCap uint64, queueLimit int, cloc
 				pool.lock.Lock()
 				pool.connectedQueue.Refresh()
 				pool.lock.Unlock()
-			case stop := <-pool.stopCh:
+			case <-clock.After(persistCumTimeRefresh):
-				close(stop)
+				pool.ndb.setCumTime(pool.logOffset(clock.Now()))
+			case <-pool.stopCh:
 				return
 			}
 		}
@@ -172,13 +196,12 @@ func newClientPool(db ethdb.Database, freeClientCap uint64, queueLimit int, cloc
 
 // stop shuts the client pool down
 func (f *clientPool) stop() {
-	stop := make(chan struct{})
+	close(f.stopCh)
-	f.stopCh <- stop
-	<-stop
 	f.lock.Lock()
 	f.closed = true
-	f.saveToDb()
 	f.lock.Unlock()
+	f.ndb.setCumTime(f.logOffset(f.clock.Now()))
+	f.ndb.close()
 }
 
 // connect should be called after a successful handshake. If the connection was
@@ -187,7 +210,7 @@ func (f *clientPool) connect(peer clientPeer, capacity uint64) bool {
 	f.lock.Lock()
 	defer f.lock.Unlock()
 
-	// Short circuit is clientPool is already closed.
+	// Short circuit if clientPool is already closed.
 	if f.closed {
 		return false
 	}
@@ -199,14 +222,18 @@ func (f *clientPool) connect(peer clientPeer, capacity uint64) bool {
 		return false
 	}
 	// Create a clientInfo but do not add it yet
-	now := f.clock.Now()
+	var (
-	posBalance := f.getPosBalance(id).value
+		posBalance uint64
+		negBalance uint64
+	)
+	pb := f.ndb.getOrNewPB(id)
+	posBalance = pb.value
 	e := &clientInfo{pool: f, peer: peer, address: freeID, queueIndex: -1, id: id, priority: posBalance != 0}
 
-	var negBalance uint64
+	nb := f.ndb.getOrNewNB(freeID)
-	nb := f.negBalanceMap[freeID]
+	if nb.logValue != 0 {
-	if nb != nil {
+		negBalance = uint64(math.Exp(float64(nb.logValue-f.logOffset(f.clock.Now())) / fixedPointMultiplier))
-		negBalance = uint64(math.Exp(float64(nb.logValue-f.logOffset(now)) / fixedPointMultiplier))
+		negBalance *= uint64(time.Second)
 	}
 	// If the client is a free client, assign with a low free capacity,
 	// Otherwise assign with the given value(priority client)
@@ -219,6 +246,7 @@ func (f *clientPool) connect(peer clientPeer, capacity uint64) bool {
 	}
 	e.capacity = capacity
 
+	// Starts a balance tracker
 	e.balanceTracker.init(f.clock, capacity)
 	e.balanceTracker.setBalance(posBalance, negBalance)
 	f.setClientPriceFactors(e)
@@ -228,9 +256,9 @@ func (f *clientPool) connect(peer clientPeer, capacity uint64) bool {
 	//
 	// If the priority of the newly added client is lower than the priority of
 	// all connected clients, the client is rejected.
-	newCapacity := f.connectedCapacity + capacity
+	newCapacity := f.connectedCap + capacity
 	newCount := f.connectedQueue.Size() + 1
-	if newCapacity > f.capacityLimit || newCount > f.countLimit {
+	if newCapacity > f.capLimit || newCount > f.connLimit {
 		var (
 			kickList     []*clientInfo
 			kickPriority int64
@@ -241,10 +269,16 @@ func (f *clientPool) connect(peer clientPeer, capacity uint64) bool {
 			kickPriority = priority
 			newCapacity -= c.capacity
 			newCount--
-			return newCapacity > f.capacityLimit || newCount > f.countLimit
+			return newCapacity > f.capLimit || newCount > f.connLimit
 		})
-		if newCapacity > f.capacityLimit || newCount > f.countLimit || (e.balanceTracker.estimatedPriority(now+mclock.AbsTime(connectedBias), false)-kickPriority) > 0 {
+		bias := freeConnectedBias
-			// reject client
+		if e.priority {
+			bias = priorityConnectedBias
+		}
+		if f.disableBias {
+			bias = 0
+		}
+		if newCapacity > f.capLimit || newCount > f.connLimit || (e.balanceTracker.estimatedPriority(f.clock.Now()+mclock.AbsTime(bias), false)-kickPriority) > 0 {
 			for _, c := range kickList {
 				f.connectedQueue.Push(c)
 			}
@@ -254,24 +288,19 @@ func (f *clientPool) connect(peer clientPeer, capacity uint64) bool {
 		}
 		// accept new client, drop old ones
 		for _, c := range kickList {
-			f.dropClient(c, now, true)
+			f.dropClient(c, f.clock.Now(), true)
 		}
 	}
-	// client accepted, finish setting it up
-	if nb != nil {
-		delete(f.negBalanceMap, freeID)
-		f.negBalanceQueue.Remove(nb.queueIndex)
-	}
 	if e.priority {
 		e.balanceTracker.addCallback(balanceCallbackZero, 0, func() { f.balanceExhausted(id) })
 	}
 	f.connectedMap[id] = e
 	f.connectedQueue.Push(e)
-	f.connectedCapacity += e.capacity
+	f.connectedCap += e.capacity
-	totalConnectedGauge.Update(int64(f.connectedCapacity))
 	if e.capacity != f.freeClientCap {
 		e.peer.updateCapacity(e.capacity)
 	}
+	totalConnectedGauge.Update(int64(f.connectedCap))
 	clientConnectedMeter.Mark(1)
 	log.Debug("Client accepted", "address", freeID)
 	return true
@@ -287,12 +316,11 @@ func (f *clientPool) disconnect(p clientPeer) {
 	if f.closed {
 		return
 	}
-	address := p.freeClientId()
 	id := p.ID()
 	// Short circuit if the peer hasn't been registered.
 	e := f.connectedMap[id]
 	if e == nil {
-		log.Debug("Client not connected", "address", address, "id", peerIdToString(id))
+		log.Debug("Client not connected", "address", p.freeClientId(), "id", peerIdToString(id))
 		return
 	}
 	f.dropClient(e, f.clock.Now(), false)
@@ -307,8 +335,8 @@ func (f *clientPool) dropClient(e *clientInfo, now mclock.AbsTime, kick bool) {
 	f.finalizeBalance(e, now)
 	f.connectedQueue.Remove(e.queueIndex)
 	delete(f.connectedMap, e.id)
-	f.connectedCapacity -= e.capacity
+	f.connectedCap -= e.capacity
-	totalConnectedGauge.Update(int64(f.connectedCapacity))
+	totalConnectedGauge.Update(int64(f.connectedCap))
 	if kick {
 		clientKickedMeter.Mark(1)
 		log.Debug("Client kicked out", "address", e.address)
@@ -324,18 +352,17 @@ func (f *clientPool) dropClient(e *clientInfo, now mclock.AbsTime, kick bool) {
 func (f *clientPool) finalizeBalance(c *clientInfo, now mclock.AbsTime) {
 	c.balanceTracker.stop(now)
 	pos, neg := c.balanceTracker.getBalance(now)
-	pb := f.getPosBalance(c.id)
+
+	pb, nb := f.ndb.getOrNewPB(c.id), f.ndb.getOrNewNB(c.address)
 	pb.value = pos
-	f.storePosBalance(pb)
+	f.ndb.setPB(c.id, pb)
-	if neg < 1 {
+
-		neg = 1
+	neg /= uint64(time.Second)
-	}
+	if neg > 1 {
-	nb := &negBalance{address: c.address, queueIndex: -1, logValue: int64(math.Log(float64(neg))*fixedPointMultiplier) + f.logOffset(now)}
+		nb.logValue = int64(math.Log(float64(neg))*fixedPointMultiplier) + f.logOffset(now)
-	f.negBalanceMap[c.address] = nb
+		f.ndb.setNB(c.address, nb)
-	f.negBalanceQueue.Push(nb, -nb.logValue)
+	} else {
-	if f.negBalanceQueue.Size() > f.queueLimit {
+		f.ndb.delNB(c.address) // Negative balance is small enough, drop it directly.
-		nn := f.negBalanceQueue.PopItem().(*negBalance)
-		delete(f.negBalanceMap, nn.address)
 	}
 }
 
@@ -351,8 +378,8 @@ func (f *clientPool) balanceExhausted(id enode.ID) {
 	}
 	c.priority = false
 	if c.capacity != f.freeClientCap {
-		f.connectedCapacity += f.freeClientCap - c.capacity
+		f.connectedCap += f.freeClientCap - c.capacity
-		totalConnectedGauge.Update(int64(f.connectedCapacity))
+		totalConnectedGauge.Update(int64(f.connectedCap))
 		c.capacity = f.freeClientCap
 		c.peer.updateCapacity(c.capacity)
 	}
@@ -360,18 +387,16 @@ func (f *clientPool) balanceExhausted(id enode.ID) {
 
 // setConnLimit sets the maximum number and total capacity of connected clients,
 // dropping some of them if necessary.
-func (f *clientPool) setLimits(count int, totalCap uint64) {
+func (f *clientPool) setLimits(totalConn int, totalCap uint64) {
 	f.lock.Lock()
 	defer f.lock.Unlock()
 
-	f.countLimit = count
+	f.connLimit = totalConn
-	f.capacityLimit = totalCap
+	f.capLimit = totalCap
-	if f.connectedCapacity > f.capacityLimit || f.connectedQueue.Size() > f.countLimit {
+	if f.connectedCap > f.capLimit || f.connectedQueue.Size() > f.connLimit {
-		now := mclock.Now()
 		f.connectedQueue.MultiPop(func(data interface{}, priority int64) bool {
-			c := data.(*clientInfo)
+			f.dropClient(data.(*clientInfo), mclock.Now(), true)
-			f.dropClient(c, now, true)
+			return f.connectedCap > f.capLimit || f.connectedQueue.Size() > f.connLimit
-			return f.connectedCapacity > f.capacityLimit || f.connectedQueue.Size() > f.countLimit
 		})
 	}
 }
@@ -390,11 +415,14 @@ func (f *clientPool) requestCost(p *peer, cost uint64) {
 
 // logOffset calculates the time-dependent offset for the logarithmic
 // representation of negative balance
+//
+// From another point of view, the result returned by the function represents
+// the total time that the clientpool is cumulatively running(total_minutes/multiplier).
 func (f *clientPool) logOffset(now mclock.AbsTime) int64 {
 	// Note: fixedPointMultiplier acts as a multiplier here; the reason for dividing the divisor
 	// is to avoid int64 overflow. We assume that int64(negBalanceExpTC) >> fixedPointMultiplier.
-	logDecay := int64((time.Duration(now - f.startupTime)) / (negBalanceExpTC / fixedPointMultiplier))
+	cumTime := int64((time.Duration(now - f.startTime)) / (negBalanceExpTC / fixedPointMultiplier))
-	return f.logOffsetAtStartup + logDecay
+	return f.startCumTime + cumTime
 }
 
 // setPriceFactors changes pricing factors for both positive and negative balances.
@@ -415,100 +443,6 @@ func (f *clientPool) setClientPriceFactors(c *clientInfo) {
 	c.balanceTracker.setFactors(false, f.posFactors.timeFactor+float64(c.capacity)*f.posFactors.capacityFactor/1000000, f.posFactors.requestFactor)
 }
 
-// clientPoolStorage is the RLP representation of the pool's database storage
-type clientPoolStorage struct {
-	LogOffset uint64
-	List      []*negBalance
-}
-
-// loadFromDb restores pool status from the database storage
-// (automatically called at initialization)
-func (f *clientPool) loadFromDb() {
-	enc, err := f.db.Get(clientPoolDbKey)
-	if err != nil {
-		return
-	}
-	var storage clientPoolStorage
-	err = rlp.DecodeBytes(enc, &storage)
-	if err != nil {
-		log.Error("Failed to decode client list", "err", err)
-		return
-	}
-	f.logOffsetAtStartup = int64(storage.LogOffset)
-	f.startupTime = f.clock.Now()
-	for _, e := range storage.List {
-		log.Debug("Loaded free client record", "address", e.address, "logValue", e.logValue)
-		f.negBalanceMap[e.address] = e
-		f.negBalanceQueue.Push(e, -e.logValue)
-	}
-}
-
-// saveToDb saves pool status to the database storage
-// (automatically called during shutdown)
-func (f *clientPool) saveToDb() {
-	now := f.clock.Now()
-	storage := clientPoolStorage{
-		LogOffset: uint64(f.logOffset(now)),
-	}
-	for _, c := range f.connectedMap {
-		f.finalizeBalance(c, now)
-	}
-	i := 0
-	storage.List = make([]*negBalance, len(f.negBalanceMap))
-	for _, e := range f.negBalanceMap {
-		storage.List[i] = e
-		i++
-	}
-	enc, err := rlp.EncodeToBytes(storage)
-	if err != nil {
-		log.Error("Failed to encode negative balance list", "err", err)
-	} else {
-		f.db.Put(clientPoolDbKey, enc)
-	}
-}
-
-// storePosBalance stores a single positive balance entry in the database
-func (f *clientPool) storePosBalance(b *posBalance) {
-	if b.value == b.lastStored {
-		return
-	}
-	enc, err := rlp.EncodeToBytes(b)
-	if err != nil {
-		log.Error("Failed to encode client balance", "err", err)
-	} else {
-		f.db.Put(append(clientBalanceDbKey, b.id[:]...), enc)
-		b.lastStored = b.value
-	}
-}
-
-// getPosBalance retrieves a single positive balance entry from cache or the database
-func (f *clientPool) getPosBalance(id enode.ID) *posBalance {
-	if b, ok := f.posBalanceMap[id]; ok {
-		f.posBalanceQueue.Remove(b.queueIndex)
-		f.posBalanceAccessCounter--
-		f.posBalanceQueue.Push(b, f.posBalanceAccessCounter)
-		return b
-	}
-	balance := &posBalance{}
-	if enc, err := f.db.Get(append(clientBalanceDbKey, id[:]...)); err == nil {
-		if err := rlp.DecodeBytes(enc, balance); err != nil {
-			log.Error("Failed to decode client balance", "err", err)
-			balance = &posBalance{}
-		}
-	}
-	balance.id = id
-	balance.queueIndex = -1
-	if f.posBalanceQueue.Size() >= f.queueLimit {
-		b := f.posBalanceQueue.PopItem().(*posBalance)
-		f.storePosBalance(b)
-		delete(f.posBalanceMap, b.id)
-	}
-	f.posBalanceAccessCounter--
-	f.posBalanceQueue.Push(balance, f.posBalanceAccessCounter)
-	f.posBalanceMap[id] = balance
-	return balance
-}
-
 // addBalance updates the positive balance of a client.
 // If setTotal is false then the given amount is added to the balance.
 // If setTotal is true then amount represents the total amount ever added to the
@@ -518,11 +452,18 @@ func (f *clientPool) addBalance(id enode.ID, amount uint64, setTotal bool) {
 	f.lock.Lock()
 	defer f.lock.Unlock()
 
-	pb := f.getPosBalance(id)
+	pb := f.ndb.getOrNewPB(id)
 	c := f.connectedMap[id]
-	var negBalance uint64
 	if c != nil {
-		pb.value, negBalance = c.balanceTracker.getBalance(f.clock.Now())
+		posBalance, negBalance := c.balanceTracker.getBalance(f.clock.Now())
+		pb.value = posBalance
+		defer func() {
+			c.balanceTracker.setBalance(pb.value, negBalance)
+			if !c.priority && pb.value > 0 {
+				c.priority = true
+				c.balanceTracker.addCallback(balanceCallbackZero, 0, func() { f.balanceExhausted(id) })
+			}
+		}()
 	}
 	if setTotal {
 		if pb.value+amount > pb.lastTotal {
@@ -535,21 +476,12 @@ func (f *clientPool) addBalance(id enode.ID, amount uint64, setTotal bool) {
 		pb.value += amount
 		pb.lastTotal += amount
 	}
-	f.storePosBalance(pb)
+	f.ndb.setPB(id, pb)
-	if c != nil {
-		c.balanceTracker.setBalance(pb.value, negBalance)
-		if !c.priority && pb.value > 0 {
-			c.priority = true
-			c.balanceTracker.addCallback(balanceCallbackZero, 0, func() { f.balanceExhausted(id) })
-		}
-	}
 }
 
 // posBalance represents a recently accessed positive balance entry
 type posBalance struct {
-	id                           enode.ID
+	value, lastTotal uint64
-	value, lastStored, lastTotal uint64
-	queueIndex                   int // position in posBalanceQueue
 }
 
 // EncodeRLP implements rlp.Encoder
@@ -566,44 +498,200 @@ func (e *posBalance) DecodeRLP(s *rlp.Stream) error {
 		return err
 	}
 	e.value = entry.Value
-	e.lastStored = entry.Value
 	e.lastTotal = entry.LastTotal
 	return nil
 }
 
-// posSetIndex callback updates posBalance item index in posBalanceQueue
-func posSetIndex(a interface{}, index int) {
-	a.(*posBalance).queueIndex = index
-}
-
 // negBalance represents a negative balance entry of a disconnected client
-type negBalance struct {
+type negBalance struct{ logValue int64 }
-	address    string
-	logValue   int64
-	queueIndex int // position in negBalanceQueue
-}
 
 // EncodeRLP implements rlp.Encoder
 func (e *negBalance) EncodeRLP(w io.Writer) error {
-	return rlp.Encode(w, []interface{}{e.address, uint64(e.logValue)})
+	return rlp.Encode(w, []interface{}{uint64(e.logValue)})
 }
 
 // DecodeRLP implements rlp.Decoder
 func (e *negBalance) DecodeRLP(s *rlp.Stream) error {
 	var entry struct {
-		Address  string
 		LogValue uint64
 	}
 	if err := s.Decode(&entry); err != nil {
 		return err
 	}
-	e.address = entry.Address
 	e.logValue = int64(entry.LogValue)
-	e.queueIndex = -1
 	return nil
 }
 
-// negSetIndex callback updates negBalance item index in negBalanceQueue
+const (
-func negSetIndex(a interface{}, index int) {
+	// nodeDBVersion is the version identifier of the node data in db
-	a.(*negBalance).queueIndex = index
+	nodeDBVersion = 0
+
+	// dbCleanupCycle is the cycle of db for useless data cleanup
+	dbCleanupCycle = time.Hour
+)
+
+var (
+	positiveBalancePrefix    = []byte("pb:")      // dbVersion(uint16 big endian) + positiveBalancePrefix + id -> balance
+	negativeBalancePrefix    = []byte("nb:")      // dbVersion(uint16 big endian) + negativeBalancePrefix + ip -> balance
+	cumulativeRunningTimeKey = []byte("cumTime:") // dbVersion(uint16 big endian) + cumulativeRunningTimeKey -> cumTime
+)
+
+type nodeDB struct {
+	db              ethdb.Database
+	pcache          *lru.Cache
+	ncache          *lru.Cache
+	auxbuf          []byte                                // 37-byte auxiliary buffer for key encoding
+	verbuf          [2]byte                               // 2-byte auxiliary buffer for db version
+	nbEvictCallBack func(mclock.AbsTime, negBalance) bool // Callback to determine whether the negative balance can be evicted.
+	clock           mclock.Clock
+	closeCh         chan struct{}
+}
+
+func newNodeDB(db ethdb.Database, clock mclock.Clock) *nodeDB {
+	pcache, _ := lru.New(posBalanceCacheLimit)
+	ncache, _ := lru.New(negBalanceCacheLimit)
+	ndb := &nodeDB{
+		db:      db,
+		pcache:  pcache,
+		ncache:  ncache,
+		auxbuf:  make([]byte, 37),
+		clock:   clock,
+		closeCh: make(chan struct{}),
+	}
+	binary.BigEndian.PutUint16(ndb.verbuf[:], uint16(nodeDBVersion))
+	go ndb.expirer()
+	return ndb
+}
+
+func (db *nodeDB) close() {
+	close(db.closeCh)
+}
+
+func (db *nodeDB) key(id []byte, neg bool) []byte {
+	prefix := positiveBalancePrefix
+	if neg {
+		prefix = negativeBalancePrefix
+	}
+	db.auxbuf = db.auxbuf[:0]
+	copy(db.auxbuf[:len(db.verbuf)], db.verbuf[:])
+	copy(db.auxbuf[len(db.verbuf):len(db.verbuf)+len(prefix)], prefix)
+	copy(db.auxbuf[len(prefix)+len(db.verbuf):len(prefix)+len(db.verbuf)+len(id)], id)
+	return db.auxbuf[:len(prefix)+len(db.verbuf)+len(id)]
+}
+
+func (db *nodeDB) getCumTime() int64 {
+	blob, err := db.db.Get(append(cumulativeRunningTimeKey, db.verbuf[:]...))
+	if err != nil || len(blob) == 0 {
+		return 0
+	}
+	return int64(binary.BigEndian.Uint64(blob))
+}
+
+func (db *nodeDB) setCumTime(v int64) {
+	binary.BigEndian.PutUint64(db.auxbuf[:8], uint64(v))
+	db.db.Put(append(cumulativeRunningTimeKey, db.verbuf[:]...), db.auxbuf[:8])
+}
+
+func (db *nodeDB) getOrNewPB(id enode.ID) posBalance {
+	key := db.key(id.Bytes(), false)
+	item, exist := db.pcache.Get(string(key))
+	if exist {
+		return item.(posBalance)
+	}
+	var balance posBalance
+	if enc, err := db.db.Get(key); err == nil {
+		if err := rlp.DecodeBytes(enc, &balance); err != nil {
+			log.Error("Failed to decode positive balance", "err", err)
+		}
+	}
+	db.pcache.Add(string(key), balance)
+	return balance
+}
+
+func (db *nodeDB) setPB(id enode.ID, b posBalance) {
+	key := db.key(id.Bytes(), false)
+	enc, err := rlp.EncodeToBytes(&(b))
+	if err != nil {
+		log.Error("Failed to encode positive balance", "err", err)
+		return
+	}
+	db.db.Put(key, enc)
+	db.pcache.Add(string(key), b)
+}
+
+func (db *nodeDB) delPB(id enode.ID) {
+	key := db.key(id.Bytes(), false)
+	db.db.Delete(key)
+	db.pcache.Remove(string(key))
+}
+
+func (db *nodeDB) getOrNewNB(id string) negBalance {
+	key := db.key([]byte(id), true)
+	item, exist := db.ncache.Get(string(key))
+	if exist {
+		return item.(negBalance)
+	}
+	var balance negBalance
+	if enc, err := db.db.Get(key); err == nil {
+		if err := rlp.DecodeBytes(enc, &balance); err != nil {
+			log.Error("Failed to decode negative balance", "err", err)
+		}
+	}
+	db.ncache.Add(string(key), balance)
+	return balance
+}
+
+func (db *nodeDB) setNB(id string, b negBalance) {
+	key := db.key([]byte(id), true)
+	enc, err := rlp.EncodeToBytes(&(b))
+	if err != nil {
+		log.Error("Failed to encode negative balance", "err", err)
+		return
+	}
+	db.db.Put(key, enc)
+	db.ncache.Add(string(key), b)
+}
+
+func (db *nodeDB) delNB(id string) {
+	key := db.key([]byte(id), true)
+	db.db.Delete(key)
+	db.ncache.Remove(string(key))
+}
+
+func (db *nodeDB) expirer() {
+	for {
+		select {
+		case <-db.clock.After(dbCleanupCycle):
+			db.expireNodes()
+		case <-db.closeCh:
+			return
+		}
+	}
+}
+
+// expireNodes iterates the whole node db and checks whether the negative balance
+// entry can deleted.
+//
+// The rationale behind this is: server doesn't need to keep the negative balance
+// records if they are low enough.
+func (db *nodeDB) expireNodes() {
+	var (
+		visited int
+		deleted int
+		start   = time.Now()
+	)
+	iter := db.db.NewIteratorWithPrefix(append(db.verbuf[:], negativeBalancePrefix...))
+	for iter.Next() {
+		visited += 1
+		var balance negBalance
+		if err := rlp.DecodeBytes(iter.Value(), &balance); err != nil {
+			log.Error("Failed to decode negative balance", "err", err)
+			continue
+		}
+		if db.nbEvictCallBack != nil && db.nbEvictCallBack(db.clock.Now(), balance) {
+			deleted += 1
+			db.db.Delete(iter.Key())
+		}
+	}
+	log.Debug("Expire nodes", "visited", visited, "deleted", deleted, "elapsed", common.PrettyDuration(time.Since(start)))
 }

les/clientpool_test.go

@@ -17,8 +17,11 @@
 package les
 
 import (
+	"bytes"
 	"fmt"
+	"math"
 	"math/rand"
+	"reflect"
 	"testing"
 	"time"
 
@@ -76,8 +79,9 @@ func testClientPool(t *testing.T, connLimit, clientCount, paidCount int, randomD
 		disconnFn = func(id enode.ID) {
 			disconnCh <- int(id[0]) + int(id[1])<<8
 		}
-		pool = newClientPool(db, 1, 10000, &clock, disconnFn)
+		pool = newClientPool(db, 1, &clock, disconnFn)
 	)
+	pool.disableBias = true
 	pool.setLimits(connLimit, uint64(connLimit))
 	pool.setPriceFactors(priceFactors{1, 0, 1}, priceFactors{1, 0, 1})
 
@@ -89,16 +93,9 @@ func testClientPool(t *testing.T, connLimit, clientCount, paidCount int, randomD
 			t.Fatalf("Test peer #%d rejected", i)
 		}
 	}
-	// since all accepted peers are new and should not be kicked out, the next one should be rejected
-	if pool.connect(poolTestPeer(connLimit), 0) {
-		connected[connLimit] = true
-		t.Fatalf("Peer accepted over connected limit")
-	}
-
 	// randomly connect and disconnect peers, expect to have a similar total connection time at the end
 	for tickCounter := 0; tickCounter < testClientPoolTicks; tickCounter++ {
 		clock.Run(1 * time.Second)
-		//time.Sleep(time.Microsecond * 100)
 
 		if tickCounter == testClientPoolTicks/4 {
 			// give a positive balance to some of the peers
@@ -157,24 +154,329 @@ func testClientPool(t *testing.T, connLimit, clientCount, paidCount int, randomD
 			t.Errorf("Total connected time of test node #%d (%d) outside expected range (%d to %d)", i, connTicks[i], min, max)
 		}
 	}
-
-	// a previously unknown peer should be accepted now
-	if !pool.connect(poolTestPeer(54321), 0) {
-		t.Fatalf("Previously unknown peer rejected")
-	}
-
-	// close and restart pool
-	pool.stop()
-	pool = newClientPool(db, 1, 10000, &clock, func(id enode.ID) {})
-	pool.setLimits(connLimit, uint64(connLimit))
-
-	// try connecting all known peers (connLimit should be filled up)
-	for i := 0; i < clientCount; i++ {
-		pool.connect(poolTestPeer(i), 0)
-	}
-	// expect pool to remember known nodes and kick out one of them to accept a new one
-	if !pool.connect(poolTestPeer(54322), 0) {
-		t.Errorf("Previously unknown peer rejected after restarting pool")
-	}
 	pool.stop()
 }
+
+func TestConnectPaidClient(t *testing.T) {
+	var (
+		clock mclock.Simulated
+		db    = rawdb.NewMemoryDatabase()
+	)
+	pool := newClientPool(db, 1, &clock, nil)
+	defer pool.stop()
+	pool.setLimits(10, uint64(10))
+	pool.setPriceFactors(priceFactors{1, 0, 1}, priceFactors{1, 0, 1})
+
+	// Add balance for an external client and mark it as paid client
+	pool.addBalance(poolTestPeer(0).ID(), 1000, false)
+
+	if !pool.connect(poolTestPeer(0), 10) {
+		t.Fatalf("Failed to connect paid client")
+	}
+}
+
+func TestConnectPaidClientToSmallPool(t *testing.T) {
+	var (
+		clock mclock.Simulated
+		db    = rawdb.NewMemoryDatabase()
+	)
+	pool := newClientPool(db, 1, &clock, nil)
+	defer pool.stop()
+	pool.setLimits(10, uint64(10)) // Total capacity limit is 10
+	pool.setPriceFactors(priceFactors{1, 0, 1}, priceFactors{1, 0, 1})
+
+	// Add balance for an external client and mark it as paid client
+	pool.addBalance(poolTestPeer(0).ID(), 1000, false)
+
+	// Connect a fat paid client to pool, should reject it.
+	if pool.connect(poolTestPeer(0), 100) {
+		t.Fatalf("Connected fat paid client, should reject it")
+	}
+}
+
+func TestConnectPaidClientToFullPool(t *testing.T) {
+	var (
+		clock mclock.Simulated
+		db    = rawdb.NewMemoryDatabase()
+	)
+	removeFn := func(enode.ID) {} // Noop
+	pool := newClientPool(db, 1, &clock, removeFn)
+	defer pool.stop()
+	pool.setLimits(10, uint64(10)) // Total capacity limit is 10
+	pool.setPriceFactors(priceFactors{1, 0, 1}, priceFactors{1, 0, 1})
+
+	for i := 0; i < 10; i++ {
+		pool.addBalance(poolTestPeer(i).ID(), 1000000000, false)
+		pool.connect(poolTestPeer(i), 1)
+	}
+	pool.addBalance(poolTestPeer(11).ID(), 1000, false) // Add low balance to new paid client
+	if pool.connect(poolTestPeer(11), 1) {
+		t.Fatalf("Low balance paid client should be rejected")
+	}
+	clock.Run(time.Second)
+	pool.addBalance(poolTestPeer(12).ID(), 1000000000*60, false) // Add high balance to new paid client
+	if !pool.connect(poolTestPeer(12), 1) {
+		t.Fatalf("High balance paid client should be accpected")
+	}
+}
+
+func TestPaidClientKickedOut(t *testing.T) {
+	var (
+		clock    mclock.Simulated
+		db       = rawdb.NewMemoryDatabase()
+		kickedCh = make(chan int, 1)
+	)
+	removeFn := func(id enode.ID) { kickedCh <- int(id[0]) }
+	pool := newClientPool(db, 1, &clock, removeFn)
+	defer pool.stop()
+	pool.setLimits(10, uint64(10)) // Total capacity limit is 10
+	pool.setPriceFactors(priceFactors{1, 0, 1}, priceFactors{1, 0, 1})
+
+	for i := 0; i < 10; i++ {
+		pool.addBalance(poolTestPeer(i).ID(), 1000000000, false) // 1 second allowance
+		pool.connect(poolTestPeer(i), 1)
+		clock.Run(time.Millisecond)
+	}
+	clock.Run(time.Second)
+	clock.Run(freeConnectedBias)
+	if !pool.connect(poolTestPeer(11), 0) {
+		t.Fatalf("Free client should be accectped")
+	}
+	select {
+	case id := <-kickedCh:
+		if id != 0 {
+			t.Fatalf("Kicked client mismatch, want %v, got %v", 0, id)
+		}
+	case <-time.NewTimer(time.Second).C:
+		t.Fatalf("timeout")
+	}
+}
+
+func TestConnectFreeClient(t *testing.T) {
+	var (
+		clock mclock.Simulated
+		db    = rawdb.NewMemoryDatabase()
+	)
+	pool := newClientPool(db, 1, &clock, nil)
+	defer pool.stop()
+	pool.setLimits(10, uint64(10))
+	pool.setPriceFactors(priceFactors{1, 0, 1}, priceFactors{1, 0, 1})
+	if !pool.connect(poolTestPeer(0), 10) {
+		t.Fatalf("Failed to connect free client")
+	}
+}
+
+func TestConnectFreeClientToFullPool(t *testing.T) {
+	var (
+		clock mclock.Simulated
+		db    = rawdb.NewMemoryDatabase()
+	)
+	removeFn := func(enode.ID) {} // Noop
+	pool := newClientPool(db, 1, &clock, removeFn)
+	defer pool.stop()
+	pool.setLimits(10, uint64(10)) // Total capacity limit is 10
+	pool.setPriceFactors(priceFactors{1, 0, 1}, priceFactors{1, 0, 1})
+
+	for i := 0; i < 10; i++ {
+		pool.connect(poolTestPeer(i), 1)
+	}
+	if pool.connect(poolTestPeer(11), 1) {
+		t.Fatalf("New free client should be rejected")
+	}
+	clock.Run(time.Minute)
+	if pool.connect(poolTestPeer(12), 1) {
+		t.Fatalf("New free client should be rejected")
+	}
+	clock.Run(time.Millisecond)
+	clock.Run(4 * time.Minute)
+	if !pool.connect(poolTestPeer(13), 1) {
+		t.Fatalf("Old client connects more than 5min should be kicked")
+	}
+}
+
+func TestFreeClientKickedOut(t *testing.T) {
+	var (
+		clock  mclock.Simulated
+		db     = rawdb.NewMemoryDatabase()
+		kicked = make(chan int, 10)
+	)
+	removeFn := func(id enode.ID) { kicked <- int(id[0]) }
+	pool := newClientPool(db, 1, &clock, removeFn)
+	defer pool.stop()
+	pool.setLimits(10, uint64(10)) // Total capacity limit is 10
+	pool.setPriceFactors(priceFactors{1, 0, 1}, priceFactors{1, 0, 1})
+
+	for i := 0; i < 10; i++ {
+		pool.connect(poolTestPeer(i), 1)
+		clock.Run(time.Millisecond)
+	}
+	if pool.connect(poolTestPeer(11), 1) {
+		t.Fatalf("New free client should be rejected")
+	}
+	clock.Run(5 * time.Minute)
+	for i := 0; i < 10; i++ {
+		pool.connect(poolTestPeer(i+10), 1)
+	}
+	for i := 0; i < 10; i++ {
+		select {
+		case id := <-kicked:
+			if id != i {
+				t.Fatalf("Kicked client mismatch, want %v, got %v", i, id)
+			}
+		case <-time.NewTimer(time.Second).C:
+			t.Fatalf("timeout")
+		}
+	}
+}
+
+func TestPositiveBalanceCalculation(t *testing.T) {
+	var (
+		clock  mclock.Simulated
+		db     = rawdb.NewMemoryDatabase()
+		kicked = make(chan int, 10)
+	)
+	removeFn := func(id enode.ID) { kicked <- int(id[0]) } // Noop
+	pool := newClientPool(db, 1, &clock, removeFn)
+	defer pool.stop()
+	pool.setLimits(10, uint64(10)) // Total capacity limit is 10
+	pool.setPriceFactors(priceFactors{1, 0, 1}, priceFactors{1, 0, 1})
+
+	pool.addBalance(poolTestPeer(0).ID(), uint64(time.Minute*3), false)
+	pool.connect(poolTestPeer(0), 10)
+	clock.Run(time.Minute)
+
+	pool.disconnect(poolTestPeer(0))
+	pb := pool.ndb.getOrNewPB(poolTestPeer(0).ID())
+	if pb.value != uint64(time.Minute*2) {
+		t.Fatalf("Positive balance mismatch, want %v, got %v", uint64(time.Minute*2), pb.value)
+	}
+}
+
+func TestNegativeBalanceCalculation(t *testing.T) {
+	var (
+		clock  mclock.Simulated
+		db     = rawdb.NewMemoryDatabase()
+		kicked = make(chan int, 10)
+	)
+	removeFn := func(id enode.ID) { kicked <- int(id[0]) } // Noop
+	pool := newClientPool(db, 1, &clock, removeFn)
+	defer pool.stop()
+	pool.setLimits(10, uint64(10)) // Total capacity limit is 10
+	pool.setPriceFactors(priceFactors{1, 0, 1}, priceFactors{1, 0, 1})
+
+	for i := 0; i < 10; i++ {
+		pool.connect(poolTestPeer(i), 1)
+	}
+	clock.Run(time.Second)
+
+	for i := 0; i < 10; i++ {
+		pool.disconnect(poolTestPeer(i))
+		nb := pool.ndb.getOrNewNB(poolTestPeer(i).freeClientId())
+		if nb.logValue != 0 {
+			t.Fatalf("Short connection shouldn't be recorded")
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		pool.connect(poolTestPeer(i), 1)
+	}
+	clock.Run(time.Minute)
+	for i := 0; i < 10; i++ {
+		pool.disconnect(poolTestPeer(i))
+		nb := pool.ndb.getOrNewNB(poolTestPeer(i).freeClientId())
+		nb.logValue -= pool.logOffset(clock.Now())
+		nb.logValue /= fixedPointMultiplier
+		if nb.logValue != int64(math.Log(float64(time.Minute/time.Second))) {
+			t.Fatalf("Negative balance mismatch, want %v, got %v", int64(math.Log(float64(time.Minute/time.Second))), nb.logValue)
+		}
+	}
+}
+
+func TestNodeDB(t *testing.T) {
+	ndb := newNodeDB(rawdb.NewMemoryDatabase(), mclock.System{})
+	defer ndb.close()
+
+	if !bytes.Equal(ndb.verbuf[:], []byte{0x00, 0x00}) {
+		t.Fatalf("version buffer mismatch, want %v, got %v", []byte{0x00, 0x00}, ndb.verbuf)
+	}
+	var cases = []struct {
+		id       enode.ID
+		ip       string
+		balance  interface{}
+		positive bool
+	}{
+		{enode.ID{0x00, 0x01, 0x02}, "", posBalance{value: 100, lastTotal: 200}, true},
+		{enode.ID{0x00, 0x01, 0x02}, "", posBalance{value: 200, lastTotal: 300}, true},
+		{enode.ID{}, "127.0.0.1", negBalance{logValue: 10}, false},
+		{enode.ID{}, "127.0.0.1", negBalance{logValue: 20}, false},
+	}
+	for _, c := range cases {
+		if c.positive {
+			ndb.setPB(c.id, c.balance.(posBalance))
+			if pb := ndb.getOrNewPB(c.id); !reflect.DeepEqual(pb, c.balance.(posBalance)) {
+				t.Fatalf("Positive balance mismatch, want %v, got %v", c.balance.(posBalance), pb)
+			}
+		} else {
+			ndb.setNB(c.ip, c.balance.(negBalance))
+			if nb := ndb.getOrNewNB(c.ip); !reflect.DeepEqual(nb, c.balance.(negBalance)) {
+				t.Fatalf("Negative balance mismatch, want %v, got %v", c.balance.(negBalance), nb)
+			}
+		}
+	}
+	for _, c := range cases {
+		if c.positive {
+			ndb.delPB(c.id)
+			if pb := ndb.getOrNewPB(c.id); !reflect.DeepEqual(pb, posBalance{}) {
+				t.Fatalf("Positive balance mismatch, want %v, got %v", posBalance{}, pb)
+			}
+		} else {
+			ndb.delNB(c.ip)
+			if nb := ndb.getOrNewNB(c.ip); !reflect.DeepEqual(nb, negBalance{}) {
+				t.Fatalf("Negative balance mismatch, want %v, got %v", negBalance{}, nb)
+			}
+		}
+	}
+	ndb.setCumTime(100)
+	if ndb.getCumTime() != 100 {
+		t.Fatalf("Cumulative time mismatch, want %v, got %v", 100, ndb.getCumTime())
+	}
+}
+
+func TestNodeDBExpiration(t *testing.T) {
+	var iterated int
+	callback := func(now mclock.AbsTime, b negBalance) bool {
+		iterated += 1
+		return true
+	}
+	clock := &mclock.Simulated{}
+	ndb := newNodeDB(rawdb.NewMemoryDatabase(), clock)
+	defer ndb.close()
+	ndb.nbEvictCallBack = callback
+
+	var cases = []struct {
+		ip      string
+		balance negBalance
+	}{
+		{"127.0.0.1", negBalance{logValue: 10}},
+		{"127.0.0.2", negBalance{logValue: 10}},
+		{"127.0.0.3", negBalance{logValue: 10}},
+		{"127.0.0.4", negBalance{logValue: 10}},
+	}
+	for _, c := range cases {
+		ndb.setNB(c.ip, c.balance)
+	}
+	clock.Run(time.Hour + time.Minute)
+	time.Sleep(300 * time.Millisecond)
+	if iterated != 4 {
+		t.Fatalf("Failed to evict useless negative balances, want %v, got %d", 4, iterated)
+	}
+
+	for _, c := range cases {
+		ndb.setNB(c.ip, c.balance)
+	}
+	clock.Run(time.Hour + time.Minute)
+	time.Sleep(300 * time.Millisecond)
+	if iterated != 8 {
+		t.Fatalf("Failed to evict useless negative balances, want %v, got %d", 4, iterated)
+	}
+}

les/server.go

@@ -113,7 +113,7 @@ func NewLesServer(e *eth.Ethereum, config *eth.Config) (*LesServer, error) {
 		maxCapacity = totalRecharge
 	}
 	srv.fcManager.SetCapacityLimits(srv.freeCapacity, maxCapacity, srv.freeCapacity*2)
-	srv.clientPool = newClientPool(srv.chainDb, srv.freeCapacity, 10000, mclock.System{}, func(id enode.ID) { go srv.peers.Unregister(peerIdToString(id)) })
+	srv.clientPool = newClientPool(srv.chainDb, srv.freeCapacity, mclock.System{}, func(id enode.ID) { go srv.peers.Unregister(peerIdToString(id)) })
 	srv.clientPool.setPriceFactors(priceFactors{0, 1, 1}, priceFactors{0, 1, 1})
 
 	checkpoint := srv.latestLocalCheckpoint()
@@ -183,9 +183,9 @@ func (s *LesServer) Stop() {
 	s.peers.Close()
 
 	s.fcManager.Stop()
-	s.clientPool.stop()
 	s.costTracker.stop()
 	s.handler.stop()
+	s.clientPool.stop() // client pool should be closed after handler.
 	s.servingQueue.stop()
 
 	// Note, bloom trie indexer is closed by parent bloombits indexer.

les/test_helper.go

@@ -280,7 +280,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da
 	}
 	server.costTracker, server.freeCapacity = newCostTracker(db, server.config)
 	server.costTracker.testCostList = testCostList(0) // Disable flow control mechanism.
-	server.clientPool = newClientPool(db, 1, 10000, clock, nil)
+	server.clientPool = newClientPool(db, 1, clock, nil)
 	server.clientPool.setLimits(10000, 10000) // Assign enough capacity for clientpool
 	server.handler = newServerHandler(server, simulation.Blockchain(), db, txpool, func() bool { return true })
 	if server.oracle != nil {