From fce218f7f83cbee98781f7f81adce1cd69222128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 31 Aug 2015 20:21:02 +0300 Subject: [PATCH] core, eth: split the db blocks into headers and bodies --- core/chain_manager.go | 7 +- core/chain_util.go | 158 ++++++++++++++++++++++++++++++++++++------ core/types/block.go | 4 ++ eth/backend.go | 108 ++++++++++++----------------- 4 files changed, 185 insertions(+), 92 deletions(-) diff --git a/core/chain_manager.go b/core/chain_manager.go index c8127951ea..8dccd04c55 100644 --- a/core/chain_manager.go +++ b/core/chain_manager.go @@ -361,12 +361,7 @@ func (bc *ChainManager) Genesis() *types.Block { // Block fetching methods func (bc *ChainManager) HasBlock(hash common.Hash) bool { - if bc.cache.Contains(hash) { - return true - } - - data, _ := bc.chainDb.Get(append(blockHashPre, hash[:]...)) - return len(data) != 0 + return bc.GetBlock(hash) != nil } func (self *ChainManager) GetBlockHashesFromHash(hash common.Hash, max uint64) (chain []common.Hash) { diff --git a/core/chain_util.go b/core/chain_util.go index 84b462ce3d..19ea243a45 100644 --- a/core/chain_util.go +++ b/core/chain_util.go @@ -19,7 +19,6 @@ package core import ( "bytes" "math/big" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -30,9 +29,12 @@ import ( ) var ( - blockHashPre = []byte("block-hash-") + headerHashPre = []byte("header-hash-") + bodyHashPre = []byte("body-hash-") blockNumPre = []byte("block-num-") ExpDiffPeriod = big.NewInt(100000) + + blockHashPre = []byte("block-hash-") // [deprecated by eth/63] ) // CalcDifficulty is the difficulty adjustment algorithm. It returns @@ -112,27 +114,91 @@ func CalcGasLimit(parent *types.Block) *big.Int { return gl } -// GetBlockByHash returns the block corresponding to the hash or nil if not found -func GetBlockByHash(db common.Database, hash common.Hash) *types.Block { - data, _ := db.Get(append(blockHashPre, hash[:]...)) +// storageBody is the block body encoding used for the database. +type storageBody struct { + Transactions []*types.Transaction + Uncles []*types.Header +} + +// GetHeaderRLPByHash retrieves a block header in its raw RLP database encoding, +// or nil if the header's not found. +func GetHeaderRLPByHash(db common.Database, hash common.Hash) []byte { + data, _ := db.Get(append(headerHashPre, hash[:]...)) + return data +} + +// GetHeaderByHash retrieves the block header corresponding to the hash, nil if +// none found. +func GetHeaderByHash(db common.Database, hash common.Hash) *types.Header { + data := GetHeaderRLPByHash(db, hash) if len(data) == 0 { return nil } - var block types.StorageBlock - if err := rlp.Decode(bytes.NewReader(data), &block); err != nil { - glog.V(logger.Error).Infof("invalid block RLP for hash %x: %v", hash, err) + header := new(types.Header) + if err := rlp.Decode(bytes.NewReader(data), header); err != nil { + glog.V(logger.Error).Infof("invalid block header RLP for hash %x: %v", hash, err) return nil } - return (*types.Block)(&block) + return header } -// GetBlockByHash returns the canonical block by number or nil if not found +// GetBodyRLPByHash retrieves the block body (transactions and uncles) in RLP +// encoding, and the associated total difficulty. +func GetBodyRLPByHash(db common.Database, hash common.Hash) ([]byte, *big.Int) { + combo, _ := db.Get(append(bodyHashPre, hash[:]...)) + if len(combo) == 0 { + return nil, nil + } + buffer := bytes.NewBuffer(combo) + + td := new(big.Int) + if err := rlp.Decode(buffer, td); err != nil { + glog.V(logger.Error).Infof("invalid block td RLP for hash %x: %v", hash, err) + return nil, nil + } + return buffer.Bytes(), td +} + +// GetBodyByHash retrieves the block body (transactons, uncles, total difficulty) +// corresponding to the hash, nils if none found. +func GetBodyByHash(db common.Database, hash common.Hash) ([]*types.Transaction, []*types.Header, *big.Int) { + data, td := GetBodyRLPByHash(db, hash) + if len(data) == 0 || td == nil { + return nil, nil, nil + } + body := new(storageBody) + if err := rlp.Decode(bytes.NewReader(data), body); err != nil { + glog.V(logger.Error).Infof("invalid block body RLP for hash %x: %v", hash, err) + return nil, nil, nil + } + return body.Transactions, body.Uncles, td +} + +// GetBlockByHash retrieves an entire block corresponding to the hash, assembling +// it back from the stored header and body. +func GetBlockByHash(db common.Database, hash common.Hash) *types.Block { + // Retrieve the block header and body contents + header := GetHeaderByHash(db, hash) + if header == nil { + return nil + } + transactions, uncles, td := GetBodyByHash(db, hash) + if td == nil { + return nil + } + // Reassemble the block and return + block := types.NewBlockWithHeader(header).WithBody(transactions, uncles) + block.Td = td + + return block +} + +// GetBlockByNumber returns the canonical block by number or nil if not found. func GetBlockByNumber(db common.Database, number uint64) *types.Block { key, _ := db.Get(append(blockNumPre, big.NewInt(int64(number)).Bytes()...)) if len(key) == 0 { return nil } - return GetBlockByHash(db, common.BytesToHash(key)) } @@ -159,21 +225,67 @@ func WriteHead(db common.Database, block *types.Block) error { return nil } -// WriteBlock writes a block to the database -func WriteBlock(db common.Database, block *types.Block) error { - tstart := time.Now() - - enc, _ := rlp.EncodeToBytes((*types.StorageBlock)(block)) - key := append(blockHashPre, block.Hash().Bytes()...) - err := db.Put(key, enc) +// WriteHeader serializes a block header into the database. +func WriteHeader(db common.Database, header *types.Header) error { + data, err := rlp.EncodeToBytes(header) if err != nil { - glog.Fatal("db write fail:", err) return err } - - if glog.V(logger.Debug) { - glog.Infof("wrote block #%v %s. Took %v\n", block.Number(), common.PP(block.Hash().Bytes()), time.Since(tstart)) + key := append(headerHashPre, header.Hash().Bytes()...) + if err := db.Put(key, data); err != nil { + glog.Fatalf("failed to store header into database: %v", err) + return err } - + glog.V(logger.Debug).Infof("stored header #%v [%x…]", header.Number, header.Hash().Bytes()[:4]) return nil } + +// WriteBody serializes the body of a block into the database. +func WriteBody(db common.Database, block *types.Block) error { + body, err := rlp.EncodeToBytes(&storageBody{block.Transactions(), block.Uncles()}) + if err != nil { + return err + } + td, err := rlp.EncodeToBytes(block.Td) + if err != nil { + return err + } + key := append(bodyHashPre, block.Hash().Bytes()...) + if err := db.Put(key, append(td, body...)); err != nil { + glog.Fatalf("failed to store block body into database: %v", err) + return err + } + glog.V(logger.Debug).Infof("stored block body #%v [%x…]", block.Number, block.Hash().Bytes()[:4]) + return nil +} + +// WriteBlock serializes a block into the database, header and body separately. +func WriteBlock(db common.Database, block *types.Block) error { + // Store the body first to retain database consistency + if err := WriteBody(db, block); err != nil { + return err + } + // Store the header too, signaling full block ownership + if err := WriteHeader(db, block.Header()); err != nil { + return err + } + return nil +} + +// [deprecated by eth/63] +// GetBlockByHashOld returns the old combined block corresponding to the hash +// or nil if not found. This method is only used by the upgrade mechanism to +// access the old combined block representation. It will be dropped after the +// network transitions to eth/63. +func GetBlockByHashOld(db common.Database, hash common.Hash) *types.Block { + data, _ := db.Get(append(blockHashPre, hash[:]...)) + if len(data) == 0 { + return nil + } + var block types.StorageBlock + if err := rlp.Decode(bytes.NewReader(data), &block); err != nil { + glog.V(logger.Error).Infof("invalid block RLP for hash %x: %v", hash, err) + return nil + } + return (*types.Block)(&block) +} diff --git a/core/types/block.go b/core/types/block.go index fd81db04cd..558b46e010 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -135,6 +135,7 @@ type Block struct { ReceivedAt time.Time } +// [deprecated by eth/63] // StorageBlock defines the RLP encoding of a Block stored in the // state database. The StorageBlock encoding contains fields that // would otherwise need to be recomputed. @@ -147,6 +148,7 @@ type extblock struct { Uncles []*Header } +// [deprecated by eth/63] // "storage" block encoding. used for database. type storageblock struct { Header *Header @@ -268,6 +270,7 @@ func (b *Block) EncodeRLP(w io.Writer) error { }) } +// [deprecated by eth/63] func (b *StorageBlock) DecodeRLP(s *rlp.Stream) error { var sb storageblock if err := s.Decode(&sb); err != nil { @@ -277,6 +280,7 @@ func (b *StorageBlock) DecodeRLP(s *rlp.Stream) error { return nil } +// [deprecated by eth/63] func (b *StorageBlock) EncodeRLP(w io.Writer) error { return rlp.Encode(w, storageblock{ Header: b.header, diff --git a/eth/backend.go b/eth/backend.go index ad2a2c1f94..7bafcbc83e 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -18,6 +18,7 @@ package eth import ( + "bytes" "crypto/ecdsa" "encoding/json" "fmt" @@ -267,11 +268,7 @@ func New(config *Config) (*Ethereum, error) { newdb = func(path string) (common.Database, error) { return ethdb.NewLDBDatabase(path, config.DatabaseCache) } } - // attempt to merge database together, upgrading from an old version - if err := mergeDatabases(config.DataDir, newdb); err != nil { - return nil, err - } - + // Open the chain database and perform any upgrades needed chainDb, err := newdb(filepath.Join(config.DataDir, "chaindata")) if err != nil { return nil, fmt.Errorf("blockchain db err: %v", err) @@ -279,6 +276,10 @@ func New(config *Config) (*Ethereum, error) { if db, ok := chainDb.(*ethdb.LDBDatabase); ok { db.Meter("eth/db/chaindata/") } + if err := upgradeChainDatabase(chainDb); err != nil { + return nil, err + } + dappDb, err := newdb(filepath.Join(config.DataDir, "dapp")) if err != nil { return nil, fmt.Errorf("dapp db err: %v", err) @@ -718,74 +719,55 @@ func saveBlockchainVersion(db common.Database, bcVersion int) { } } -// mergeDatabases when required merge old database layout to one single database -func mergeDatabases(datadir string, newdb func(path string) (common.Database, error)) error { - // Check if already upgraded - data := filepath.Join(datadir, "chaindata") - if _, err := os.Stat(data); !os.IsNotExist(err) { +// upgradeChainDatabase ensures that the chain database stores block split into +// separate header and body entries. +func upgradeChainDatabase(db common.Database) error { + // Short circuit if the head block is stored already as separate header and body + data, err := db.Get([]byte("LastBlock")) + if err != nil { return nil } - // make sure it's not just a clean path - chainPath := filepath.Join(datadir, "blockchain") - if _, err := os.Stat(chainPath); os.IsNotExist(err) { + head := common.BytesToHash(data) + + if block := core.GetBlockByHashOld(db, head); block == nil { return nil } - glog.Infoln("Database upgrade required. Upgrading...") + // At least some of the database is still the old format, upgrade (skip the head block!) + glog.V(logger.Info).Info("Old database detected, upgrading...") - database, err := newdb(data) - if err != nil { - return fmt.Errorf("creating data db err: %v", err) - } - defer database.Close() + if db, ok := db.(*ethdb.LDBDatabase); ok { + blockPrefix := []byte("block-hash-") + for it := db.NewIterator(); it.Next(); { + // Skip anything other than a combined block + if !bytes.HasPrefix(it.Key(), blockPrefix) { + continue + } + // Skip the head block (merge last to signal upgrade completion) + if bytes.HasSuffix(it.Key(), head.Bytes()) { + continue + } + // Load the block, split and serialize (order!) + block := core.GetBlockByHashOld(db, common.BytesToHash(bytes.TrimPrefix(it.Key(), blockPrefix))) - // Migrate blocks - chainDb, err := newdb(chainPath) - if err != nil { - return fmt.Errorf("state db err: %v", err) - } - defer chainDb.Close() - - if chain, ok := chainDb.(*ethdb.LDBDatabase); ok { - glog.Infoln("Merging blockchain database...") - it := chain.NewIterator() - for it.Next() { - database.Put(it.Key(), it.Value()) + if err := core.WriteBody(db, block); err != nil { + return err + } + if err := core.WriteHeader(db, block.Header()); err != nil { + return err + } + if err := db.Delete(it.Key()); err != nil { + return err + } } - it.Release() - } + // Lastly, upgrade the head block, disabling the upgrade mechanism + current := core.GetBlockByHashOld(db, head) - // Migrate state - stateDb, err := newdb(filepath.Join(datadir, "state")) - if err != nil { - return fmt.Errorf("state db err: %v", err) - } - defer stateDb.Close() - - if state, ok := stateDb.(*ethdb.LDBDatabase); ok { - glog.Infoln("Merging state database...") - it := state.NewIterator() - for it.Next() { - database.Put(it.Key(), it.Value()) + if err := core.WriteBody(db, current); err != nil { + return err } - it.Release() - } - - // Migrate transaction / receipts - extraDb, err := newdb(filepath.Join(datadir, "extra")) - if err != nil { - return fmt.Errorf("state db err: %v", err) - } - defer extraDb.Close() - - if extra, ok := extraDb.(*ethdb.LDBDatabase); ok { - glog.Infoln("Merging transaction database...") - - it := extra.NewIterator() - for it.Next() { - database.Put(it.Key(), it.Value()) + if err := core.WriteHeader(db, current.Header()); err != nil { + return err } - it.Release() } - return nil }