From 615145e08448468b04ab11ed37b27c4bf4a06d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 26 Jan 2018 12:20:42 +0200 Subject: [PATCH] trie: count preimages in memuse, commit in KB batches --- common/size.go | 27 +++++++++++++++++++-------- internal/ethapi/api.go | 2 +- trie/database.go | 35 +++++++++++++++++++++++------------ 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/common/size.go b/common/size.go index c5a0cb0f2d..bd0fc85c7d 100644 --- a/common/size.go +++ b/common/size.go @@ -20,18 +20,29 @@ import ( "fmt" ) +// StorageSize is a wrapper around a float value that supports user friendly +// formatting. type StorageSize float64 -func (self StorageSize) String() string { - if self > 1000000 { - return fmt.Sprintf("%.2f mB", self/1000000) - } else if self > 1000 { - return fmt.Sprintf("%.2f kB", self/1000) +// String implements the stringer interface. +func (s StorageSize) String() string { + if s > 1000000 { + return fmt.Sprintf("%.2f mB", s/1000000) + } else if s > 1000 { + return fmt.Sprintf("%.2f kB", s/1000) } else { - return fmt.Sprintf("%.2f B", self) + return fmt.Sprintf("%.2f B", s) } } -func (self StorageSize) Int64() int64 { - return int64(self) +// TerminalString implements log.TerminalStringer, formatting a string for console +// output during logging. +func (s StorageSize) TerminalString() string { + if s > 1000000 { + return fmt.Sprintf("%.2fmB", s/1000000) + } else if s > 1000 { + return fmt.Sprintf("%.2fkB", s/1000) + } else { + return fmt.Sprintf("%.2fB", s) + } } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a4cba7a4db..314086335c 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -808,7 +808,7 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx "difficulty": (*hexutil.Big)(head.Difficulty), "totalDifficulty": (*hexutil.Big)(s.b.GetTd(b.Hash())), "extraData": hexutil.Bytes(head.Extra), - "size": hexutil.Uint64(uint64(b.Size().Int64())), + "size": hexutil.Uint64(b.Size()), "gasLimit": hexutil.Uint64(head.GasLimit), "gasUsed": hexutil.Uint64(head.GasUsed), "timestamp": (*hexutil.Big)(head.Time), diff --git a/trie/database.go b/trie/database.go index fb7d5d57bd..055a40911e 100644 --- a/trie/database.go +++ b/trie/database.go @@ -54,7 +54,9 @@ type Database struct { gcnodes uint64 // Nodes garbage collected since last commit gcsize common.StorageSize // Data storage garbage collected since last commit - size common.StorageSize // Storage size of the memory cache + nodesSize common.StorageSize // Storage size of the nodes cache + preimagesSize common.StorageSize // Storage size of the preimages cache + lock sync.RWMutex } @@ -101,7 +103,7 @@ func (db *Database) insert(hash common.Hash, blob []byte) { blob: common.CopyBytes(blob), children: make(map[common.Hash]int), } - db.size += common.StorageSize(common.HashLength + len(blob)) + db.nodesSize += common.StorageSize(common.HashLength + len(blob)) } // insertPreimage writes a new trie node pre-image to the memory database if it's @@ -113,6 +115,7 @@ func (db *Database) insertPreimage(hash common.Hash, preimage []byte) { return } db.preimages[hash] = common.CopyBytes(preimage) + db.preimagesSize += common.StorageSize(common.HashLength + len(preimage)) } // Node retrieves a cached trie node from memory. If it cannot be found cached, @@ -198,15 +201,15 @@ func (db *Database) Dereference(child common.Hash, parent common.Hash) { db.lock.Lock() defer db.lock.Unlock() - nodes, storage, start := len(db.nodes), db.size, time.Now() + nodes, storage, start := len(db.nodes), db.nodesSize, time.Now() db.dereference(child, parent) db.gcnodes += uint64(nodes - len(db.nodes)) - db.gcsize += storage - db.size + db.gcsize += storage - db.nodesSize db.gctime += time.Since(start) - log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.size, "time", time.Since(start), - "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.size) + log.Debug("Dereferenced trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), + "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize) } // dereference is the private locked version of Dereference. @@ -230,7 +233,7 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { db.dereference(hash, child) } delete(db.nodes, child) - db.size -= common.StorageSize(common.HashLength + len(node.blob)) + db.nodesSize -= common.StorageSize(common.HashLength + len(node.blob)) } } @@ -255,9 +258,15 @@ func (db *Database) Commit(node common.Hash) error { db.lock.RUnlock() return err } + if batch.ValueSize() > ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + return err + } + batch = db.diskdb.NewBatch() + } } // Move the trie itself into the batch, flushing if enough data is accumulated - nodes, storage := len(db.nodes), db.size + nodes, storage := len(db.nodes), db.nodesSize+db.preimagesSize if err := db.commit(node, &batch); err != nil { log.Error("Failed to commit trie from trie database", "err", err) db.lock.RUnlock() @@ -276,10 +285,12 @@ func (db *Database) Commit(node common.Hash) error { defer db.lock.Unlock() db.preimages = make(map[common.Hash][]byte) + db.preimagesSize = 0 + db.uncache(node) - log.Info("Persisted trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.size, "time", time.Since(start), - "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.size) + log.Info("Persisted trie from memory database", "nodes", nodes-len(db.nodes), "size", storage-db.nodesSize, "time", time.Since(start), + "gcnodes", db.gcnodes, "gcsize", db.gcsize, "gctime", db.gctime, "livenodes", len(db.nodes), "livesize", db.nodesSize) // Reset the garbage collection statistics db.gcnodes, db.gcsize, db.gctime = 0, 0, 0 @@ -327,7 +338,7 @@ func (db *Database) uncache(hash common.Hash) { db.uncache(child) } delete(db.nodes, hash) - db.size -= common.StorageSize(common.HashLength + len(node.blob)) + db.nodesSize -= common.StorageSize(common.HashLength + len(node.blob)) } // Size returns the current storage size of the memory cache in front of the @@ -336,5 +347,5 @@ func (db *Database) Size() common.StorageSize { db.lock.RLock() defer db.lock.RUnlock() - return db.size + return db.nodesSize + db.preimagesSize }