core, ethdb: Removed old memory database and use leveldb in-memory db

Removed the old memory database but kept the method around, which now
returns an instance of LDBDatabase with a in-memory leveldb.

Also exposes new database interfaces such as the Reader, Writer and
ReadWriter. All core database functions now take a ReadWriter instead of
a Database.
This commit is contained in:
Jeffrey Wilcke 2016-05-26 00:48:14 +02:00
parent fdd61b83ff
commit 3ca6340117
4 changed files with 155 additions and 156 deletions

View file

@ -20,7 +20,6 @@ import (
"bytes"
"encoding/binary"
"encoding/json"
"fmt"
"math/big"
"github.com/ethereum/go-ethereum/common"
@ -56,7 +55,7 @@ var (
)
// GetCanonicalHash retrieves a hash assigned to a canonical block number.
func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash {
func GetCanonicalHash(db ethdb.ReadWriter, number uint64) common.Hash {
data, _ := db.Get(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...))
if len(data) == 0 {
return common.Hash{}
@ -69,7 +68,7 @@ func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash {
// last block hash is only updated upon a full block import, the last header
// hash is updated already at header import, allowing head tracking for the
// light synchronization mechanism.
func GetHeadHeaderHash(db ethdb.Database) common.Hash {
func GetHeadHeaderHash(db ethdb.ReadWriter) common.Hash {
data, _ := db.Get(headHeaderKey)
if len(data) == 0 {
return common.Hash{}
@ -78,7 +77,7 @@ func GetHeadHeaderHash(db ethdb.Database) common.Hash {
}
// GetHeadBlockHash retrieves the hash of the current canonical head block.
func GetHeadBlockHash(db ethdb.Database) common.Hash {
func GetHeadBlockHash(db ethdb.ReadWriter) common.Hash {
data, _ := db.Get(headBlockKey)
if len(data) == 0 {
return common.Hash{}
@ -90,7 +89,7 @@ func GetHeadBlockHash(db ethdb.Database) common.Hash {
// fast synchronization. The difference between this and GetHeadBlockHash is that
// whereas the last block hash is only updated upon a full block import, the last
// fast hash is updated when importing pre-processed blocks.
func GetHeadFastBlockHash(db ethdb.Database) common.Hash {
func GetHeadFastBlockHash(db ethdb.ReadWriter) common.Hash {
data, _ := db.Get(headFastKey)
if len(data) == 0 {
return common.Hash{}
@ -100,14 +99,14 @@ func GetHeadFastBlockHash(db ethdb.Database) common.Hash {
// GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil
// if the header's not found.
func GetHeaderRLP(db ethdb.Database, hash common.Hash) rlp.RawValue {
func GetHeaderRLP(db ethdb.ReadWriter, hash common.Hash) rlp.RawValue {
data, _ := db.Get(append(append(blockPrefix, hash[:]...), headerSuffix...))
return data
}
// GetHeader retrieves the block header corresponding to the hash, nil if none
// found.
func GetHeader(db ethdb.Database, hash common.Hash) *types.Header {
func GetHeader(db ethdb.ReadWriter, hash common.Hash) *types.Header {
data := GetHeaderRLP(db, hash)
if len(data) == 0 {
return nil
@ -121,14 +120,14 @@ func GetHeader(db ethdb.Database, hash common.Hash) *types.Header {
}
// GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding.
func GetBodyRLP(db ethdb.Database, hash common.Hash) rlp.RawValue {
func GetBodyRLP(db ethdb.ReadWriter, hash common.Hash) rlp.RawValue {
data, _ := db.Get(append(append(blockPrefix, hash[:]...), bodySuffix...))
return data
}
// GetBody retrieves the block body (transactons, uncles) corresponding to the
// hash, nil if none found.
func GetBody(db ethdb.Database, hash common.Hash) *types.Body {
func GetBody(db ethdb.ReadWriter, hash common.Hash) *types.Body {
data := GetBodyRLP(db, hash)
if len(data) == 0 {
return nil
@ -143,7 +142,7 @@ func GetBody(db ethdb.Database, hash common.Hash) *types.Body {
// GetTd retrieves a block's total difficulty corresponding to the hash, nil if
// none found.
func GetTd(db ethdb.Database, hash common.Hash) *big.Int {
func GetTd(db ethdb.ReadWriter, hash common.Hash) *big.Int {
data, _ := db.Get(append(append(blockPrefix, hash.Bytes()...), tdSuffix...))
if len(data) == 0 {
return nil
@ -158,7 +157,7 @@ func GetTd(db ethdb.Database, hash common.Hash) *big.Int {
// GetBlock retrieves an entire block corresponding to the hash, assembling it
// back from the stored header and body.
func GetBlock(db ethdb.Database, hash common.Hash) *types.Block {
func GetBlock(db ethdb.ReadWriter, hash common.Hash) *types.Block {
// Retrieve the block header and body contents
header := GetHeader(db, hash)
if header == nil {
@ -174,7 +173,7 @@ func GetBlock(db ethdb.Database, hash common.Hash) *types.Block {
// GetBlockReceipts retrieves the receipts generated by the transactions included
// in a block given by its hash.
func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts {
func GetBlockReceipts(db ethdb.ReadWriter, hash common.Hash) types.Receipts {
data, _ := db.Get(append(blockReceiptsPrefix, hash[:]...))
if len(data) == 0 {
return nil
@ -193,7 +192,7 @@ func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts {
// GetTransaction retrieves a specific transaction from the database, along with
// its added positional metadata.
func GetTransaction(db ethdb.Database, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) {
func GetTransaction(db ethdb.ReadWriter, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) {
// Retrieve the transaction itself from the database
data, _ := db.Get(hash.Bytes())
if len(data) == 0 {
@ -220,7 +219,7 @@ func GetTransaction(db ethdb.Database, hash common.Hash) (*types.Transaction, co
}
// GetReceipt returns a receipt by hash
func GetReceipt(db ethdb.Database, txHash common.Hash) *types.Receipt {
func GetReceipt(db ethdb.ReadWriter, txHash common.Hash) *types.Receipt {
data, _ := db.Get(append(receiptsPrefix, txHash[:]...))
if len(data) == 0 {
return nil
@ -234,7 +233,7 @@ func GetReceipt(db ethdb.Database, txHash common.Hash) *types.Receipt {
}
// WriteCanonicalHash stores the canonical hash for the given block number.
func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) error {
func WriteCanonicalHash(db ethdb.ReadWriter, hash common.Hash, number uint64) error {
key := append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)
if err := db.Put(key, hash.Bytes()); err != nil {
glog.Fatalf("failed to store number to hash mapping into database: %v", err)
@ -244,7 +243,7 @@ func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) erro
}
// WriteHeadHeaderHash stores the head header's hash.
func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error {
func WriteHeadHeaderHash(db ethdb.ReadWriter, hash common.Hash) error {
if err := db.Put(headHeaderKey, hash.Bytes()); err != nil {
glog.Fatalf("failed to store last header's hash into database: %v", err)
return err
@ -253,7 +252,7 @@ func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error {
}
// WriteHeadBlockHash stores the head block's hash.
func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error {
func WriteHeadBlockHash(db ethdb.ReadWriter, hash common.Hash) error {
if err := db.Put(headBlockKey, hash.Bytes()); err != nil {
glog.Fatalf("failed to store last block's hash into database: %v", err)
return err
@ -262,7 +261,7 @@ func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error {
}
// WriteHeadFastBlockHash stores the fast head block's hash.
func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error {
func WriteHeadFastBlockHash(db ethdb.ReadWriter, hash common.Hash) error {
if err := db.Put(headFastKey, hash.Bytes()); err != nil {
glog.Fatalf("failed to store last fast block's hash into database: %v", err)
return err
@ -271,7 +270,7 @@ func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error {
}
// WriteHeader serializes a block header into the database.
func WriteHeader(db ethdb.Database, header *types.Header) error {
func WriteHeader(db ethdb.ReadWriter, header *types.Header) error {
data, err := rlp.EncodeToBytes(header)
if err != nil {
return err
@ -286,7 +285,7 @@ func WriteHeader(db ethdb.Database, header *types.Header) error {
}
// WriteBody serializes the body of a block into the database.
func WriteBody(db ethdb.Database, hash common.Hash, body *types.Body) error {
func WriteBody(db ethdb.ReadWriter, hash common.Hash, body *types.Body) error {
data, err := rlp.EncodeToBytes(body)
if err != nil {
return err
@ -301,7 +300,7 @@ func WriteBody(db ethdb.Database, hash common.Hash, body *types.Body) error {
}
// WriteTd serializes the total difficulty of a block into the database.
func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error {
func WriteTd(db ethdb.ReadWriter, hash common.Hash, td *big.Int) error {
data, err := rlp.EncodeToBytes(td)
if err != nil {
return err
@ -316,7 +315,7 @@ func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error {
}
// WriteBlock serializes a block into the database, header and body separately.
func WriteBlock(db ethdb.Database, block *types.Block) error {
func WriteBlock(db ethdb.ReadWriter, block *types.Block) error {
// Store the body first to retain database consistency
if err := WriteBody(db, block.Hash(), block.Body()); err != nil {
return err
@ -331,7 +330,7 @@ func WriteBlock(db ethdb.Database, block *types.Block) error {
// WriteBlockReceipts stores all the transaction receipts belonging to a block
// as a single receipt slice. This is used during chain reorganisations for
// rescheduling dropped transactions.
func WriteBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Receipts) error {
func WriteBlockReceipts(db ethdb.ReadWriter, hash common.Hash, receipts types.Receipts) error {
// Convert the receipts into their storage form and serialize them
storageReceipts := make([]*types.ReceiptForStorage, len(receipts))
for i, receipt := range receipts {
@ -350,6 +349,42 @@ func WriteBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Rece
return nil
}
// WriteBlockTransactions stores the transactions associated with a specific block
// into the given database. Beside writing the transaction, the function also
// stores a metadata entry along with the transaction, detailing the position
// of this within the blockchain.
func WriteBlockTransactions(db ethdb.ReadWriter, header types.Header, transactions types.Transactions) error {
// Iterate over each transaction and encode it with its metadata
for i, tx := range transactions {
// Encode and queue up the transaction for storage
data, err := rlp.EncodeToBytes(tx)
if err != nil {
return err
}
if err := db.Put(tx.Hash().Bytes(), data); err != nil {
return err
}
// Encode and queue up the transaction metadata for storage
meta := struct {
BlockHash common.Hash
BlockIndex uint64
Index uint64
}{
BlockHash: header.Hash(),
BlockIndex: header.Number.Uint64(),
Index: uint64(i),
}
data, err = rlp.EncodeToBytes(meta)
if err != nil {
return err
}
if err := db.Put(append(tx.Hash().Bytes(), txMetaSuffix...), data); err != nil {
return err
}
}
return nil
}
// WriteTransactions stores the transactions associated with a specific block
// into the given database. Beside writing the transaction, the function also
// stores a metadata entry along with the transaction, detailing the position
@ -394,8 +429,10 @@ func WriteTransactions(db ethdb.Database, block *types.Block) error {
}
// WriteReceipts stores a batch of transaction receipts into the database.
func WriteReceipts(db ethdb.Database, receipts types.Receipts) error {
batch := db.NewBatch()
func WriteReceipts(db ethdb.ReadWriter, receipts types.Receipts) error {
// XXX Batch has been commented out because this is now used for transaction stuff as well
// if you want to use batches perhaps you should pass a batch, rather than a database?
//batch := db.NewBatch()
// Iterate over all the receipts and queue them for database injection
for _, receipt := range receipts {
@ -404,40 +441,42 @@ func WriteReceipts(db ethdb.Database, receipts types.Receipts) error {
if err != nil {
return err
}
if err := batch.Put(append(receiptsPrefix, receipt.TxHash.Bytes()...), data); err != nil {
if err := db.Put(append(receiptsPrefix, receipt.TxHash.Bytes()...), data); err != nil {
return err
}
}
// Write the scheduled data into the database
if err := batch.Write(); err != nil {
glog.Fatalf("failed to store receipts into database: %v", err)
return err
}
/*
// Write the scheduled data into the database
if err := batch.Write(); err != nil {
glog.Fatalf("failed to store receipts into database: %v", err)
return err
}
*/
return nil
}
// DeleteCanonicalHash removes the number to hash canonical mapping.
func DeleteCanonicalHash(db ethdb.Database, number uint64) {
func DeleteCanonicalHash(db ethdb.ReadWriter, number uint64) {
db.Delete(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...))
}
// DeleteHeader removes all block header data associated with a hash.
func DeleteHeader(db ethdb.Database, hash common.Hash) {
func DeleteHeader(db ethdb.ReadWriter, hash common.Hash) {
db.Delete(append(append(blockPrefix, hash.Bytes()...), headerSuffix...))
}
// DeleteBody removes all block body data associated with a hash.
func DeleteBody(db ethdb.Database, hash common.Hash) {
func DeleteBody(db ethdb.ReadWriter, hash common.Hash) {
db.Delete(append(append(blockPrefix, hash.Bytes()...), bodySuffix...))
}
// DeleteTd removes all block total difficulty data associated with a hash.
func DeleteTd(db ethdb.Database, hash common.Hash) {
func DeleteTd(db ethdb.ReadWriter, hash common.Hash) {
db.Delete(append(append(blockPrefix, hash.Bytes()...), tdSuffix...))
}
// DeleteBlock removes all block data associated with a hash.
func DeleteBlock(db ethdb.Database, hash common.Hash) {
func DeleteBlock(db ethdb.ReadWriter, hash common.Hash) {
DeleteBlockReceipts(db, hash)
DeleteHeader(db, hash)
DeleteBody(db, hash)
@ -445,18 +484,18 @@ func DeleteBlock(db ethdb.Database, hash common.Hash) {
}
// DeleteBlockReceipts removes all receipt data associated with a block hash.
func DeleteBlockReceipts(db ethdb.Database, hash common.Hash) {
func DeleteBlockReceipts(db ethdb.ReadWriter, hash common.Hash) {
db.Delete(append(blockReceiptsPrefix, hash.Bytes()...))
}
// DeleteTransaction removes all transaction data associated with a hash.
func DeleteTransaction(db ethdb.Database, hash common.Hash) {
func DeleteTransaction(db ethdb.ReadWriter, hash common.Hash) {
db.Delete(hash.Bytes())
db.Delete(append(hash.Bytes(), txMetaSuffix...))
}
// DeleteReceipt removes all receipt data associated with a transaction hash.
func DeleteReceipt(db ethdb.Database, hash common.Hash) {
func DeleteReceipt(db ethdb.ReadWriter, hash common.Hash) {
db.Delete(append(receiptsPrefix, hash.Bytes()...))
}
@ -465,7 +504,7 @@ func DeleteReceipt(db ethdb.Database, hash common.Hash) {
// or nil if not found. This method is only used by the upgrade mechanism to
// access the old combined block representation. It will be dropped after the
// network transitions to eth/63.
func GetBlockByHashOld(db ethdb.Database, hash common.Hash) *types.Block {
func GetBlockByHashOld(db ethdb.ReadWriter, hash common.Hash) *types.Block {
data, _ := db.Get(append(blockHashPrefix, hash[:]...))
if len(data) == 0 {
return nil
@ -491,8 +530,10 @@ func mipmapKey(num, level uint64) []byte {
// WriteMapmapBloom writes each address included in the receipts' logs to the
// MIP bloom bin.
func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts) error {
batch := db.NewBatch()
func WriteMipmapBloom(db ethdb.ReadWriter, number uint64, receipts types.Receipts) error {
// XXX Batch has been commented out because this is now used for transaction stuff as well
// if you want to use batches perhaps you should pass a batch, rather than a database?
//batch := db.NewBatch()
for _, level := range MIPMapLevels {
key := mipmapKey(number, level)
bloomDat, _ := db.Get(key)
@ -502,23 +543,25 @@ func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts)
bloom.Add(log.Address.Big())
}
}
batch.Put(key, bloom.Bytes())
}
if err := batch.Write(); err != nil {
return fmt.Errorf("mipmap write fail for: %d: %v", number, err)
db.Put(key, bloom.Bytes())
}
/*
if err := batch.Write(); err != nil {
return fmt.Errorf("mipmap write fail for: %d: %v", number, err)
}
*/
return nil
}
// GetMipmapBloom returns a bloom filter using the number and level as input
// parameters. For available levels see MIPMapLevels.
func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom {
func GetMipmapBloom(db ethdb.ReadWriter, number, level uint64) types.Bloom {
bloomDat, _ := db.Get(mipmapKey(number, level))
return types.BytesToBloom(bloomDat)
}
// GetBlockChainVersion reads the version number from db.
func GetBlockChainVersion(db ethdb.Database) int {
func GetBlockChainVersion(db ethdb.ReadWriter) int {
var vsn uint
enc, _ := db.Get([]byte("BlockchainVersion"))
rlp.DecodeBytes(enc, &vsn)
@ -526,13 +569,13 @@ func GetBlockChainVersion(db ethdb.Database) int {
}
// WriteBlockChainVersion writes vsn as the version number to db.
func WriteBlockChainVersion(db ethdb.Database, vsn int) {
func WriteBlockChainVersion(db ethdb.ReadWriter, vsn int) {
enc, _ := rlp.EncodeToBytes(uint(vsn))
db.Put([]byte("BlockchainVersion"), enc)
}
// WriteChainConfig writes the chain config settings to the database.
func WriteChainConfig(db ethdb.Database, hash common.Hash, cfg *ChainConfig) error {
func WriteChainConfig(db ethdb.ReadWriter, hash common.Hash, cfg *ChainConfig) error {
// short circuit and ignore if nil config. GetChainConfig
// will return a default.
if cfg == nil {
@ -548,7 +591,7 @@ func WriteChainConfig(db ethdb.Database, hash common.Hash, cfg *ChainConfig) err
}
// GetChainConfig will fetch the network settings based on the given hash.
func GetChainConfig(db ethdb.Database, hash common.Hash) (*ChainConfig, error) {
func GetChainConfig(db ethdb.ReadWriter, hash common.Hash) (*ChainConfig, error) {
jsonChainConfig, _ := db.Get(append(configPrefix, hash[:]...))
if len(jsonChainConfig) == 0 {
return nil, ChainConfigNotFoundErr

View file

@ -285,6 +285,36 @@ func (db *LDBDatabase) NewBatch() Batch {
return &ldbBatch{db: db.db, b: new(leveldb.Batch)}
}
func (db *LDBDatabase) OpenTransaction() (Transaction, error) {
tx, err := db.db.OpenTransaction()
if err != nil {
return nil, err
}
return &ldbTransaction{db: db.db, tx: tx}, nil
}
type ldbTransaction struct {
db *leveldb.DB
tx *leveldb.Transaction
}
func (t *ldbTransaction) Put(key, value []byte) error {
return t.tx.Put(key, value, nil)
}
func (t *ldbTransaction) Delete(key []byte) error {
return t.tx.Delete(key, nil)
}
func (t *ldbTransaction) Get(key []byte) ([]byte, error) {
return t.tx.Get(key, nil)
}
func (t *ldbTransaction) Commit() error {
return t.tx.Commit()
}
type ldbBatch struct {
db *leveldb.DB
b *leveldb.Batch

View file

@ -17,14 +17,35 @@
package ethdb
type Database interface {
Put(key []byte, value []byte) error
Get(key []byte) ([]byte, error)
Delete(key []byte) error
Writer
Reader
Close()
NewBatch() Batch
OpenTransaction() (Transaction, error)
}
type Batch interface {
Put(key, value []byte) error
Write() error
}
type Writer interface {
Put(key []byte, value []byte) error
Delete(key []byte) error
}
type Reader interface {
Get(key []byte) ([]byte, error)
}
type ReadWriter interface {
Reader
Writer
}
type Transaction interface {
ReadWriter
Commit() error
}

View file

@ -17,109 +17,14 @@
package ethdb
import (
"errors"
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/storage"
)
/*
* This is a test memory database. Do not use for any production it does not get persisted
*/
type MemDatabase struct {
db map[string][]byte
lock sync.RWMutex
}
func NewMemDatabase() (*MemDatabase, error) {
return &MemDatabase{
db: make(map[string][]byte),
}, nil
}
func (db *MemDatabase) Put(key []byte, value []byte) error {
db.lock.Lock()
defer db.lock.Unlock()
db.db[string(key)] = common.CopyBytes(value)
return nil
}
func (db *MemDatabase) Set(key []byte, value []byte) {
db.lock.Lock()
defer db.lock.Unlock()
db.Put(key, value)
}
func (db *MemDatabase) Get(key []byte) ([]byte, error) {
db.lock.RLock()
defer db.lock.RUnlock()
if entry, ok := db.db[string(key)]; ok {
return entry, nil
func NewMemDatabase() (*LDBDatabase, error) {
db, err := leveldb.Open(storage.NewMemStorage(), nil)
if err != nil {
return nil, err
}
return nil, errors.New("not found")
}
func (db *MemDatabase) Keys() [][]byte {
db.lock.RLock()
defer db.lock.RUnlock()
keys := [][]byte{}
for key, _ := range db.db {
keys = append(keys, []byte(key))
}
return keys
}
/*
func (db *MemDatabase) GetKeys() []*common.Key {
data, _ := db.Get([]byte("KeyRing"))
return []*common.Key{common.NewKeyFromBytes(data)}
}
*/
func (db *MemDatabase) Delete(key []byte) error {
db.lock.Lock()
defer db.lock.Unlock()
delete(db.db, string(key))
return nil
}
func (db *MemDatabase) Close() {}
func (db *MemDatabase) NewBatch() Batch {
return &memBatch{db: db}
}
type kv struct{ k, v []byte }
type memBatch struct {
db *MemDatabase
writes []kv
lock sync.RWMutex
}
func (b *memBatch) Put(key, value []byte) error {
b.lock.Lock()
defer b.lock.Unlock()
b.writes = append(b.writes, kv{common.CopyBytes(key), common.CopyBytes(value)})
return nil
}
func (b *memBatch) Write() error {
b.lock.RLock()
defer b.lock.RUnlock()
b.db.lock.Lock()
defer b.db.lock.Unlock()
for _, kv := range b.writes {
b.db.db[string(kv.k)] = kv.v
}
return nil
return &LDBDatabase{db: db, fn: "<memory>"}, nil
}