Merge pull request #841 from gzliudan/clean_db

all: clean up and properly abstract database accesses

cmd/XDC/chaincmd.go

@@ -195,6 +195,7 @@ func initGenesis(ctx *cli.Context) error {
 		if err != nil {
 			utils.Fatalf("Failed to write genesis block: %v", err)
 		}
+		chaindb.Close()
 		log.Info("Successfully wrote genesis state", "database", name, "hash", hash)
 	}
 	return nil
@@ -322,11 +323,11 @@ func importPreimages(ctx *cli.Context) error {
 	stack, _ := makeFullNode(ctx)
 	defer stack.Close()
 
-	diskdb := utils.MakeChainDatabase(ctx, stack, false)
+	db := utils.MakeChainDatabase(ctx, stack, false)
-	defer diskdb.Close()
+	defer db.Close()
 
 	start := time.Now()
-	if err := utils.ImportPreimages(diskdb, ctx.Args().First()); err != nil {
+	if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil {
 		utils.Fatalf("Export error: %v\n", err)
 	}
 	fmt.Printf("Export done in %v\n", time.Since(start))
@@ -341,11 +342,11 @@ func exportPreimages(ctx *cli.Context) error {
 	stack, _ := makeFullNode(ctx)
 	defer stack.Close()
 
-	diskdb := utils.MakeChainDatabase(ctx, stack, true)
+	db := utils.MakeChainDatabase(ctx, stack, true)
-	defer diskdb.Close()
+	defer db.Close()
 
 	start := time.Now()
-	if err := utils.ExportPreimages(diskdb, ctx.Args().First()); err != nil {
+	if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil {
 		utils.Fatalf("Export error: %v\n", err)
 	}
 	fmt.Printf("Export done in %v\n", time.Since(start))

core/genesis.go

@@ -87,7 +87,7 @@ type GenesisMismatchError struct {
 }
 
 func (e *GenesisMismatchError) Error() string {
-	return fmt.Sprintf("database already contains an incompatible genesis block (have %x, new %x)", e.Stored[:8], e.New[:8])
+	return fmt.Sprintf("database contains incompatible genesis (have %x, new %x)", e.Stored, e.New)
 }
 
 // SetupGenesisBlock writes or updates the genesis block in db.

core/rawdb/accessors_chain.go

@@ -95,7 +95,7 @@ func DeleteHeaderNumber(db ethdb.KeyValueWriter, hash common.Hash) {
 // last block hash is only updated upon a full block import, the last header
 // hash is updated already at header import, allowing head tracking for the
 // light synchronization mechanism.
-func ReadHeadHeaderHash(db ethdb.Reader) common.Hash {
+func ReadHeadHeaderHash(db ethdb.KeyValueReader) common.Hash {
 	data, _ := db.Get(headHeaderKey)
 	if len(data) == 0 {
 		return common.Hash{}
@@ -331,10 +331,16 @@ func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
 	}
 }
 
+// ReadTdRLP retrieves a block's total difficulty corresponding to the hash in RLP encoding.
+func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
+	data, _ := db.Get(headerTDKey(number, hash))
+	return data
+}
+
 // ReadTd retrieves a block's total difficulty corresponding to the hash, nil if
 // none found.
 func ReadTd(db ethdb.Reader, hash common.Hash, number uint64) *big.Int {
-	data, _ := db.Get(headerTDKey(number, hash))
+	data := ReadTdRLP(db, hash, number)
 	if len(data) == 0 {
 		return nil
 	}
@@ -572,3 +578,12 @@ func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
 	DeleteBody(db, hash, number)
 	DeleteTd(db, hash, number)
 }
+
+// deleteBlockWithoutNumber removes all block data associated with a hash, except
+// the hash to number mapping.
+func deleteBlockWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
+	DeleteReceipts(db, hash, number)
+	deleteHeaderWithoutNumber(db, hash, number)
+	DeleteBody(db, hash, number)
+	DeleteTd(db, hash, number)
+}

core/rawdb/database.go

@@ -100,6 +100,13 @@ func NewMemoryDatabase() ethdb.Database {
 	return NewDatabase(memorydb.New())
 }
 
+// NewMemoryDatabaseWithCap creates an ephemeral in-memory key-value database with
+// an initial starting capacity, but without a freezer moving immutable chain
+// segments into cold storage.
+func NewMemoryDatabaseWithCap(size int) ethdb.Database {
+	return NewDatabase(memorydb.NewWithCap(size))
+}
+
 // NewLevelDBDatabase creates a persistent key-value database without a freezer
 // moving immutable chain segments into cold storage.
 func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {

core/rawdb/schema.go

@@ -97,6 +97,11 @@ func encodeBlockNumber(number uint64) []byte {
 	return enc
 }
 
+// headerKeyPrefix = headerPrefix + num (uint64 big endian)
+func headerKeyPrefix(number uint64) []byte {
+	return append(headerPrefix, encodeBlockNumber(number)...)
+}
+
 // headerKey = headerPrefix + num (uint64 big endian) + hash
 func headerKey(number uint64, hash common.Hash) []byte {
 	return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)

core/state/statedb_test.go

@@ -28,10 +28,10 @@ import (
 	"testing"
 	"testing/quick"
 
-	check "gopkg.in/check.v1"
 	"github.com/XinFinOrg/XDPoSChain/common"
 	"github.com/XinFinOrg/XDPoSChain/core/rawdb"
 	"github.com/XinFinOrg/XDPoSChain/core/types"
+	check "gopkg.in/check.v1"
 )
 
 // Tests that updating a state trie does not leak any database writes prior to
@@ -57,10 +57,9 @@ func TestUpdateLeaks(t *testing.T) {
 	// Ensure that no data was leaked into the database
 	it := db.NewIterator(nil, nil)
 	for it.Next() {
-		key := it.Key()
+		t.Errorf("State leaked into database: %x -> %x", it.Key(), it.Value())
-		value := it.Value()
-		t.Errorf("State leaked into database: %x -> %x", key, value)
 	}
+	it.Release()
 }
 
 // Tests that no intermediate state of an object is stored into the database,
@@ -108,16 +107,16 @@ func TestIntermediateLeaks(t *testing.T) {
 	for it.Next() {
 		key := it.Key()
 		if _, err := transDb.Get(key); err != nil {
-			val, _ := finalDb.Get(key)
+			t.Errorf("entry missing from the transition database: %x -> %x", key, it.Value())
-			t.Errorf("entry missing from the transition database: %x -> %x", key, val)
 		}
 	}
+	it.Release()
+
 	it = transDb.NewIterator(nil, nil)
 	for it.Next() {
 		key := it.Key()
 		if _, err := finalDb.Get(key); err != nil {
-			val, _ := transDb.Get(key)
+			t.Errorf("extra entry in the transition database: %x -> %x", key, it.Value())
-			t.Errorf("extra entry in the transition database: %x -> %x", key, val)
 		}
 	}
 }

eth/backend.go

@@ -120,7 +120,9 @@ func New(ctx *node.ServiceContext, config *ethconfig.Config, XDCXServ *XDCx.XDCX
 	if !config.SyncMode.IsValid() {
 		return nil, fmt.Errorf("invalid sync mode %d", config.SyncMode)
 	}
-	chainDb, err := CreateDB(ctx, config, "chaindata")
+
+	// Assemble the Ethereum object
+	chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/", false)
 	if err != nil {
 		return nil, err
 	}
@@ -341,15 +343,6 @@ func makeExtraData(extra []byte) []byte {
 	return extra
 }
 
-// CreateDB creates the chain database.
-func CreateDB(ctx *node.ServiceContext, config *ethconfig.Config, name string) (ethdb.Database, error) {
-	db, err := ctx.OpenDatabase(name, config.DatabaseCache, config.DatabaseHandles, false)
-	if err != nil {
-		return nil, err
-	}
-	return db, nil
-}
-
 // CreateConsensusEngine creates the required type of consensus engine instance for an Ethereum service
 func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chainConfig *params.ChainConfig, db ethdb.Database) consensus.Engine {
 	// If delegated-proof-of-stake is requested, set it up

les/backend.go

@@ -83,7 +83,7 @@ type LightEthereum struct {
 }
 
 func New(ctx *node.ServiceContext, config *ethconfig.Config) (*LightEthereum, error) {
-	chainDb, err := eth.CreateDB(ctx, config, "lightchaindata")
+	chainDb, err := ctx.OpenDatabase("lightchaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/", false)
 	if err != nil {
 		return nil, err
 	}

light/nodeset.go

@@ -144,7 +144,7 @@ func (db *NodeSet) Store(target ethdb.KeyValueWriter) {
 	}
 }
 
-// NodeList stores an ordered list of trie nodes. It implements ethdb.Putter.
+// NodeList stores an ordered list of trie nodes. It implements ethdb.Writer.
 type NodeList []rlp.RawValue
 
 func (n NodeList) Delete(key []byte) error {

node/service.go

@@ -40,11 +40,11 @@ type ServiceContext struct {
 // OpenDatabase opens an existing database with the given name (or creates one
 // if no previous can be found) from within the node's data directory. If the
 // node is an ephemeral one, a memory database is returned.
-func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int, readonly bool) (ethdb.Database, error) {
+func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
 	if ctx.config.DataDir == "" {
 		return rawdb.NewMemoryDatabase(), nil
 	}
-	db, err := rawdb.NewLevelDBDatabase(ctx.config.resolvePath(name), cache, handles, "", readonly)
+	db, err := rawdb.NewLevelDBDatabase(ctx.config.resolvePath(name), cache, handles, namespace, readonly)
 	if err != nil {
 		return nil, err
 	}

node/service_test.go

@@ -35,7 +35,7 @@ func TestContextDatabases(t *testing.T) {
 	}
 	// Request the opening/creation of a database and ensure it persists to disk
 	ctx := &ServiceContext{config: &Config{Name: "unit-test", DataDir: dir}}
-	db, err := ctx.OpenDatabase("persistent", 0, 0, false)
+	db, err := ctx.OpenDatabase("persistent", 0, 0, "", false)
 	if err != nil {
 		t.Fatalf("failed to open persistent database: %v", err)
 	}
@@ -46,7 +46,7 @@ func TestContextDatabases(t *testing.T) {
 	}
 	// Request th opening/creation of an ephemeral database and ensure it's not persisted
 	ctx = &ServiceContext{config: &Config{DataDir: ""}}
-	db, err = ctx.OpenDatabase("ephemeral", 0, 0, false)
+	db, err = ctx.OpenDatabase("ephemeral", 0, 0, "", false)
 	if err != nil {
 		t.Fatalf("failed to open ephemeral database: %v", err)
 	}