From 3ca6340117327606a602bf01cae5e678f91aeb27 Mon Sep 17 00:00:00 2001 From: Jeffrey Wilcke Date: Thu, 26 May 2016 00:48:14 +0200 Subject: [PATCH] core, ethdb: Removed old memory database and use leveldb in-memory db Removed the old memory database but kept the method around, which now returns an instance of LDBDatabase with a in-memory leveldb. Also exposes new database interfaces such as the Reader, Writer and ReadWriter. All core database functions now take a ReadWriter instead of a Database. --- core/database_util.go | 145 +++++++++++++++++++++++++-------------- ethdb/database.go | 30 ++++++++ ethdb/interface.go | 27 +++++++- ethdb/memory_database.go | 109 ++--------------------------- 4 files changed, 155 insertions(+), 156 deletions(-) diff --git a/core/database_util.go b/core/database_util.go index 3ba80062c1..166938cf32 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -20,7 +20,6 @@ import ( "bytes" "encoding/binary" "encoding/json" - "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -56,7 +55,7 @@ var ( ) // GetCanonicalHash retrieves a hash assigned to a canonical block number. -func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash { +func GetCanonicalHash(db ethdb.ReadWriter, number uint64) common.Hash { data, _ := db.Get(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) if len(data) == 0 { return common.Hash{} @@ -69,7 +68,7 @@ func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash { // last block hash is only updated upon a full block import, the last header // hash is updated already at header import, allowing head tracking for the // light synchronization mechanism. -func GetHeadHeaderHash(db ethdb.Database) common.Hash { +func GetHeadHeaderHash(db ethdb.ReadWriter) common.Hash { data, _ := db.Get(headHeaderKey) if len(data) == 0 { return common.Hash{} @@ -78,7 +77,7 @@ func GetHeadHeaderHash(db ethdb.Database) common.Hash { } // GetHeadBlockHash retrieves the hash of the current canonical head block. -func GetHeadBlockHash(db ethdb.Database) common.Hash { +func GetHeadBlockHash(db ethdb.ReadWriter) common.Hash { data, _ := db.Get(headBlockKey) if len(data) == 0 { return common.Hash{} @@ -90,7 +89,7 @@ func GetHeadBlockHash(db ethdb.Database) common.Hash { // fast synchronization. The difference between this and GetHeadBlockHash is that // whereas the last block hash is only updated upon a full block import, the last // fast hash is updated when importing pre-processed blocks. -func GetHeadFastBlockHash(db ethdb.Database) common.Hash { +func GetHeadFastBlockHash(db ethdb.ReadWriter) common.Hash { data, _ := db.Get(headFastKey) if len(data) == 0 { return common.Hash{} @@ -100,14 +99,14 @@ func GetHeadFastBlockHash(db ethdb.Database) common.Hash { // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. -func GetHeaderRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { +func GetHeaderRLP(db ethdb.ReadWriter, hash common.Hash) rlp.RawValue { data, _ := db.Get(append(append(blockPrefix, hash[:]...), headerSuffix...)) return data } // GetHeader retrieves the block header corresponding to the hash, nil if none // found. -func GetHeader(db ethdb.Database, hash common.Hash) *types.Header { +func GetHeader(db ethdb.ReadWriter, hash common.Hash) *types.Header { data := GetHeaderRLP(db, hash) if len(data) == 0 { return nil @@ -121,14 +120,14 @@ func GetHeader(db ethdb.Database, hash common.Hash) *types.Header { } // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func GetBodyRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { +func GetBodyRLP(db ethdb.ReadWriter, hash common.Hash) rlp.RawValue { data, _ := db.Get(append(append(blockPrefix, hash[:]...), bodySuffix...)) return data } // GetBody retrieves the block body (transactons, uncles) corresponding to the // hash, nil if none found. -func GetBody(db ethdb.Database, hash common.Hash) *types.Body { +func GetBody(db ethdb.ReadWriter, hash common.Hash) *types.Body { data := GetBodyRLP(db, hash) if len(data) == 0 { return nil @@ -143,7 +142,7 @@ func GetBody(db ethdb.Database, hash common.Hash) *types.Body { // GetTd retrieves a block's total difficulty corresponding to the hash, nil if // none found. -func GetTd(db ethdb.Database, hash common.Hash) *big.Int { +func GetTd(db ethdb.ReadWriter, hash common.Hash) *big.Int { data, _ := db.Get(append(append(blockPrefix, hash.Bytes()...), tdSuffix...)) if len(data) == 0 { return nil @@ -158,7 +157,7 @@ func GetTd(db ethdb.Database, hash common.Hash) *big.Int { // GetBlock retrieves an entire block corresponding to the hash, assembling it // back from the stored header and body. -func GetBlock(db ethdb.Database, hash common.Hash) *types.Block { +func GetBlock(db ethdb.ReadWriter, hash common.Hash) *types.Block { // Retrieve the block header and body contents header := GetHeader(db, hash) if header == nil { @@ -174,7 +173,7 @@ func GetBlock(db ethdb.Database, hash common.Hash) *types.Block { // GetBlockReceipts retrieves the receipts generated by the transactions included // in a block given by its hash. -func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts { +func GetBlockReceipts(db ethdb.ReadWriter, hash common.Hash) types.Receipts { data, _ := db.Get(append(blockReceiptsPrefix, hash[:]...)) if len(data) == 0 { return nil @@ -193,7 +192,7 @@ func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts { // GetTransaction retrieves a specific transaction from the database, along with // its added positional metadata. -func GetTransaction(db ethdb.Database, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { +func GetTransaction(db ethdb.ReadWriter, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { // Retrieve the transaction itself from the database data, _ := db.Get(hash.Bytes()) if len(data) == 0 { @@ -220,7 +219,7 @@ func GetTransaction(db ethdb.Database, hash common.Hash) (*types.Transaction, co } // GetReceipt returns a receipt by hash -func GetReceipt(db ethdb.Database, txHash common.Hash) *types.Receipt { +func GetReceipt(db ethdb.ReadWriter, txHash common.Hash) *types.Receipt { data, _ := db.Get(append(receiptsPrefix, txHash[:]...)) if len(data) == 0 { return nil @@ -234,7 +233,7 @@ func GetReceipt(db ethdb.Database, txHash common.Hash) *types.Receipt { } // WriteCanonicalHash stores the canonical hash for the given block number. -func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) error { +func WriteCanonicalHash(db ethdb.ReadWriter, hash common.Hash, number uint64) error { key := append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...) if err := db.Put(key, hash.Bytes()); err != nil { glog.Fatalf("failed to store number to hash mapping into database: %v", err) @@ -244,7 +243,7 @@ func WriteCanonicalHash(db ethdb.Database, hash common.Hash, number uint64) erro } // WriteHeadHeaderHash stores the head header's hash. -func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { +func WriteHeadHeaderHash(db ethdb.ReadWriter, hash common.Hash) error { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last header's hash into database: %v", err) return err @@ -253,7 +252,7 @@ func WriteHeadHeaderHash(db ethdb.Database, hash common.Hash) error { } // WriteHeadBlockHash stores the head block's hash. -func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { +func WriteHeadBlockHash(db ethdb.ReadWriter, hash common.Hash) error { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last block's hash into database: %v", err) return err @@ -262,7 +261,7 @@ func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error { } // WriteHeadFastBlockHash stores the fast head block's hash. -func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error { +func WriteHeadFastBlockHash(db ethdb.ReadWriter, hash common.Hash) error { if err := db.Put(headFastKey, hash.Bytes()); err != nil { glog.Fatalf("failed to store last fast block's hash into database: %v", err) return err @@ -271,7 +270,7 @@ func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error { } // WriteHeader serializes a block header into the database. -func WriteHeader(db ethdb.Database, header *types.Header) error { +func WriteHeader(db ethdb.ReadWriter, header *types.Header) error { data, err := rlp.EncodeToBytes(header) if err != nil { return err @@ -286,7 +285,7 @@ func WriteHeader(db ethdb.Database, header *types.Header) error { } // WriteBody serializes the body of a block into the database. -func WriteBody(db ethdb.Database, hash common.Hash, body *types.Body) error { +func WriteBody(db ethdb.ReadWriter, hash common.Hash, body *types.Body) error { data, err := rlp.EncodeToBytes(body) if err != nil { return err @@ -301,7 +300,7 @@ func WriteBody(db ethdb.Database, hash common.Hash, body *types.Body) error { } // WriteTd serializes the total difficulty of a block into the database. -func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error { +func WriteTd(db ethdb.ReadWriter, hash common.Hash, td *big.Int) error { data, err := rlp.EncodeToBytes(td) if err != nil { return err @@ -316,7 +315,7 @@ func WriteTd(db ethdb.Database, hash common.Hash, td *big.Int) error { } // WriteBlock serializes a block into the database, header and body separately. -func WriteBlock(db ethdb.Database, block *types.Block) error { +func WriteBlock(db ethdb.ReadWriter, block *types.Block) error { // Store the body first to retain database consistency if err := WriteBody(db, block.Hash(), block.Body()); err != nil { return err @@ -331,7 +330,7 @@ func WriteBlock(db ethdb.Database, block *types.Block) error { // WriteBlockReceipts stores all the transaction receipts belonging to a block // as a single receipt slice. This is used during chain reorganisations for // rescheduling dropped transactions. -func WriteBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Receipts) error { +func WriteBlockReceipts(db ethdb.ReadWriter, hash common.Hash, receipts types.Receipts) error { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { @@ -350,6 +349,42 @@ func WriteBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Rece return nil } +// WriteBlockTransactions stores the transactions associated with a specific block +// into the given database. Beside writing the transaction, the function also +// stores a metadata entry along with the transaction, detailing the position +// of this within the blockchain. +func WriteBlockTransactions(db ethdb.ReadWriter, header types.Header, transactions types.Transactions) error { + // Iterate over each transaction and encode it with its metadata + for i, tx := range transactions { + // Encode and queue up the transaction for storage + data, err := rlp.EncodeToBytes(tx) + if err != nil { + return err + } + if err := db.Put(tx.Hash().Bytes(), data); err != nil { + return err + } + // Encode and queue up the transaction metadata for storage + meta := struct { + BlockHash common.Hash + BlockIndex uint64 + Index uint64 + }{ + BlockHash: header.Hash(), + BlockIndex: header.Number.Uint64(), + Index: uint64(i), + } + data, err = rlp.EncodeToBytes(meta) + if err != nil { + return err + } + if err := db.Put(append(tx.Hash().Bytes(), txMetaSuffix...), data); err != nil { + return err + } + } + return nil +} + // WriteTransactions stores the transactions associated with a specific block // into the given database. Beside writing the transaction, the function also // stores a metadata entry along with the transaction, detailing the position @@ -394,8 +429,10 @@ func WriteTransactions(db ethdb.Database, block *types.Block) error { } // WriteReceipts stores a batch of transaction receipts into the database. -func WriteReceipts(db ethdb.Database, receipts types.Receipts) error { - batch := db.NewBatch() +func WriteReceipts(db ethdb.ReadWriter, receipts types.Receipts) error { + // XXX Batch has been commented out because this is now used for transaction stuff as well + // if you want to use batches perhaps you should pass a batch, rather than a database? + //batch := db.NewBatch() // Iterate over all the receipts and queue them for database injection for _, receipt := range receipts { @@ -404,40 +441,42 @@ func WriteReceipts(db ethdb.Database, receipts types.Receipts) error { if err != nil { return err } - if err := batch.Put(append(receiptsPrefix, receipt.TxHash.Bytes()...), data); err != nil { + if err := db.Put(append(receiptsPrefix, receipt.TxHash.Bytes()...), data); err != nil { return err } } - // Write the scheduled data into the database - if err := batch.Write(); err != nil { - glog.Fatalf("failed to store receipts into database: %v", err) - return err - } + /* + // Write the scheduled data into the database + if err := batch.Write(); err != nil { + glog.Fatalf("failed to store receipts into database: %v", err) + return err + } + */ return nil } // DeleteCanonicalHash removes the number to hash canonical mapping. -func DeleteCanonicalHash(db ethdb.Database, number uint64) { +func DeleteCanonicalHash(db ethdb.ReadWriter, number uint64) { db.Delete(append(blockNumPrefix, big.NewInt(int64(number)).Bytes()...)) } // DeleteHeader removes all block header data associated with a hash. -func DeleteHeader(db ethdb.Database, hash common.Hash) { +func DeleteHeader(db ethdb.ReadWriter, hash common.Hash) { db.Delete(append(append(blockPrefix, hash.Bytes()...), headerSuffix...)) } // DeleteBody removes all block body data associated with a hash. -func DeleteBody(db ethdb.Database, hash common.Hash) { +func DeleteBody(db ethdb.ReadWriter, hash common.Hash) { db.Delete(append(append(blockPrefix, hash.Bytes()...), bodySuffix...)) } // DeleteTd removes all block total difficulty data associated with a hash. -func DeleteTd(db ethdb.Database, hash common.Hash) { +func DeleteTd(db ethdb.ReadWriter, hash common.Hash) { db.Delete(append(append(blockPrefix, hash.Bytes()...), tdSuffix...)) } // DeleteBlock removes all block data associated with a hash. -func DeleteBlock(db ethdb.Database, hash common.Hash) { +func DeleteBlock(db ethdb.ReadWriter, hash common.Hash) { DeleteBlockReceipts(db, hash) DeleteHeader(db, hash) DeleteBody(db, hash) @@ -445,18 +484,18 @@ func DeleteBlock(db ethdb.Database, hash common.Hash) { } // DeleteBlockReceipts removes all receipt data associated with a block hash. -func DeleteBlockReceipts(db ethdb.Database, hash common.Hash) { +func DeleteBlockReceipts(db ethdb.ReadWriter, hash common.Hash) { db.Delete(append(blockReceiptsPrefix, hash.Bytes()...)) } // DeleteTransaction removes all transaction data associated with a hash. -func DeleteTransaction(db ethdb.Database, hash common.Hash) { +func DeleteTransaction(db ethdb.ReadWriter, hash common.Hash) { db.Delete(hash.Bytes()) db.Delete(append(hash.Bytes(), txMetaSuffix...)) } // DeleteReceipt removes all receipt data associated with a transaction hash. -func DeleteReceipt(db ethdb.Database, hash common.Hash) { +func DeleteReceipt(db ethdb.ReadWriter, hash common.Hash) { db.Delete(append(receiptsPrefix, hash.Bytes()...)) } @@ -465,7 +504,7 @@ func DeleteReceipt(db ethdb.Database, hash common.Hash) { // or nil if not found. This method is only used by the upgrade mechanism to // access the old combined block representation. It will be dropped after the // network transitions to eth/63. -func GetBlockByHashOld(db ethdb.Database, hash common.Hash) *types.Block { +func GetBlockByHashOld(db ethdb.ReadWriter, hash common.Hash) *types.Block { data, _ := db.Get(append(blockHashPrefix, hash[:]...)) if len(data) == 0 { return nil @@ -491,8 +530,10 @@ func mipmapKey(num, level uint64) []byte { // WriteMapmapBloom writes each address included in the receipts' logs to the // MIP bloom bin. -func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts) error { - batch := db.NewBatch() +func WriteMipmapBloom(db ethdb.ReadWriter, number uint64, receipts types.Receipts) error { + // XXX Batch has been commented out because this is now used for transaction stuff as well + // if you want to use batches perhaps you should pass a batch, rather than a database? + //batch := db.NewBatch() for _, level := range MIPMapLevels { key := mipmapKey(number, level) bloomDat, _ := db.Get(key) @@ -502,23 +543,25 @@ func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts) bloom.Add(log.Address.Big()) } } - batch.Put(key, bloom.Bytes()) - } - if err := batch.Write(); err != nil { - return fmt.Errorf("mipmap write fail for: %d: %v", number, err) + db.Put(key, bloom.Bytes()) } + /* + if err := batch.Write(); err != nil { + return fmt.Errorf("mipmap write fail for: %d: %v", number, err) + } + */ return nil } // GetMipmapBloom returns a bloom filter using the number and level as input // parameters. For available levels see MIPMapLevels. -func GetMipmapBloom(db ethdb.Database, number, level uint64) types.Bloom { +func GetMipmapBloom(db ethdb.ReadWriter, number, level uint64) types.Bloom { bloomDat, _ := db.Get(mipmapKey(number, level)) return types.BytesToBloom(bloomDat) } // GetBlockChainVersion reads the version number from db. -func GetBlockChainVersion(db ethdb.Database) int { +func GetBlockChainVersion(db ethdb.ReadWriter) int { var vsn uint enc, _ := db.Get([]byte("BlockchainVersion")) rlp.DecodeBytes(enc, &vsn) @@ -526,13 +569,13 @@ func GetBlockChainVersion(db ethdb.Database) int { } // WriteBlockChainVersion writes vsn as the version number to db. -func WriteBlockChainVersion(db ethdb.Database, vsn int) { +func WriteBlockChainVersion(db ethdb.ReadWriter, vsn int) { enc, _ := rlp.EncodeToBytes(uint(vsn)) db.Put([]byte("BlockchainVersion"), enc) } // WriteChainConfig writes the chain config settings to the database. -func WriteChainConfig(db ethdb.Database, hash common.Hash, cfg *ChainConfig) error { +func WriteChainConfig(db ethdb.ReadWriter, hash common.Hash, cfg *ChainConfig) error { // short circuit and ignore if nil config. GetChainConfig // will return a default. if cfg == nil { @@ -548,7 +591,7 @@ func WriteChainConfig(db ethdb.Database, hash common.Hash, cfg *ChainConfig) err } // GetChainConfig will fetch the network settings based on the given hash. -func GetChainConfig(db ethdb.Database, hash common.Hash) (*ChainConfig, error) { +func GetChainConfig(db ethdb.ReadWriter, hash common.Hash) (*ChainConfig, error) { jsonChainConfig, _ := db.Get(append(configPrefix, hash[:]...)) if len(jsonChainConfig) == 0 { return nil, ChainConfigNotFoundErr diff --git a/ethdb/database.go b/ethdb/database.go index dffb42e2b0..3f603a140a 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -285,6 +285,36 @@ func (db *LDBDatabase) NewBatch() Batch { return &ldbBatch{db: db.db, b: new(leveldb.Batch)} } +func (db *LDBDatabase) OpenTransaction() (Transaction, error) { + tx, err := db.db.OpenTransaction() + if err != nil { + return nil, err + } + + return &ldbTransaction{db: db.db, tx: tx}, nil +} + +type ldbTransaction struct { + db *leveldb.DB + tx *leveldb.Transaction +} + +func (t *ldbTransaction) Put(key, value []byte) error { + return t.tx.Put(key, value, nil) +} + +func (t *ldbTransaction) Delete(key []byte) error { + return t.tx.Delete(key, nil) +} + +func (t *ldbTransaction) Get(key []byte) ([]byte, error) { + return t.tx.Get(key, nil) +} + +func (t *ldbTransaction) Commit() error { + return t.tx.Commit() +} + type ldbBatch struct { db *leveldb.DB b *leveldb.Batch diff --git a/ethdb/interface.go b/ethdb/interface.go index f4b787a52a..e26979e5bd 100644 --- a/ethdb/interface.go +++ b/ethdb/interface.go @@ -17,14 +17,35 @@ package ethdb type Database interface { - Put(key []byte, value []byte) error - Get(key []byte) ([]byte, error) - Delete(key []byte) error + Writer + Reader + Close() NewBatch() Batch + OpenTransaction() (Transaction, error) } type Batch interface { Put(key, value []byte) error Write() error } + +type Writer interface { + Put(key []byte, value []byte) error + Delete(key []byte) error +} + +type Reader interface { + Get(key []byte) ([]byte, error) +} + +type ReadWriter interface { + Reader + Writer +} + +type Transaction interface { + ReadWriter + + Commit() error +} diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index a729f52332..a7c2c3b72c 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -17,109 +17,14 @@ package ethdb import ( - "errors" - "sync" - - "github.com/ethereum/go-ethereum/common" + "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/storage" ) -/* - * This is a test memory database. Do not use for any production it does not get persisted - */ -type MemDatabase struct { - db map[string][]byte - lock sync.RWMutex -} - -func NewMemDatabase() (*MemDatabase, error) { - return &MemDatabase{ - db: make(map[string][]byte), - }, nil -} - -func (db *MemDatabase) Put(key []byte, value []byte) error { - db.lock.Lock() - defer db.lock.Unlock() - - db.db[string(key)] = common.CopyBytes(value) - return nil -} - -func (db *MemDatabase) Set(key []byte, value []byte) { - db.lock.Lock() - defer db.lock.Unlock() - - db.Put(key, value) -} - -func (db *MemDatabase) Get(key []byte) ([]byte, error) { - db.lock.RLock() - defer db.lock.RUnlock() - - if entry, ok := db.db[string(key)]; ok { - return entry, nil +func NewMemDatabase() (*LDBDatabase, error) { + db, err := leveldb.Open(storage.NewMemStorage(), nil) + if err != nil { + return nil, err } - return nil, errors.New("not found") -} - -func (db *MemDatabase) Keys() [][]byte { - db.lock.RLock() - defer db.lock.RUnlock() - - keys := [][]byte{} - for key, _ := range db.db { - keys = append(keys, []byte(key)) - } - return keys -} - -/* -func (db *MemDatabase) GetKeys() []*common.Key { - data, _ := db.Get([]byte("KeyRing")) - - return []*common.Key{common.NewKeyFromBytes(data)} -} -*/ - -func (db *MemDatabase) Delete(key []byte) error { - db.lock.Lock() - defer db.lock.Unlock() - - delete(db.db, string(key)) - return nil -} - -func (db *MemDatabase) Close() {} - -func (db *MemDatabase) NewBatch() Batch { - return &memBatch{db: db} -} - -type kv struct{ k, v []byte } - -type memBatch struct { - db *MemDatabase - writes []kv - lock sync.RWMutex -} - -func (b *memBatch) Put(key, value []byte) error { - b.lock.Lock() - defer b.lock.Unlock() - - b.writes = append(b.writes, kv{common.CopyBytes(key), common.CopyBytes(value)}) - return nil -} - -func (b *memBatch) Write() error { - b.lock.RLock() - defer b.lock.RUnlock() - - b.db.lock.Lock() - defer b.db.lock.Unlock() - - for _, kv := range b.writes { - b.db.db[string(kv.k)] = kv.v - } - return nil + return &LDBDatabase{db: db, fn: ""}, nil }