mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-08-19 18:32:23 +00:00
swarm/shed/internal: add example_dbstore_test
This commit is contained in:
Janos Guljas
parent
a307c3ab25
commit
6d6afef911
1 changed files with 280 additions and 0 deletions
280
swarm/shed/internal/example_dbstore_test.go
Normal file
280
swarm/shed/internal/example_dbstore_test.go
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
// Copyright 2018 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package internal_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/swarm/shed/internal"
|
||||
"github.com/ethereum/go-ethereum/swarm/storage"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
)
|
||||
|
||||
// DB is just an example for composing indexes.
|
||||
type DB struct {
|
||||
db *internal.DB
|
||||
|
||||
// fields and indexes
|
||||
schemaName internal.StringField
|
||||
sizeCounter internal.Uint64Field
|
||||
accessCounter internal.Uint64Field
|
||||
retrievalIndex internal.Index
|
||||
accessIndex internal.Index
|
||||
gcIndex internal.Index
|
||||
}
|
||||
|
||||
func New(path string) (db *DB, err error) {
|
||||
idb, err := internal.NewDB(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db = &DB{
|
||||
db: idb,
|
||||
}
|
||||
db.schemaName, err = idb.NewStringField("schema-name")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.sizeCounter, err = idb.NewUint64Field("size-counter")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.accessCounter, err = idb.NewUint64Field("access-counter")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.retrievalIndex, err = idb.NewIndex("Hash->StoreTimestamp|Data", internal.IndexFuncs{
|
||||
EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
|
||||
return fields.Hash, nil
|
||||
},
|
||||
DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
|
||||
e.Hash = key
|
||||
return e, nil
|
||||
},
|
||||
EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
|
||||
b := make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp))
|
||||
value = append(b, fields.Data...)
|
||||
return value, nil
|
||||
},
|
||||
DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
|
||||
e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8]))
|
||||
e.Data = value[8:]
|
||||
return e, nil
|
||||
},
|
||||
})
|
||||
db.accessIndex, err = idb.NewIndex("Hash->AccessTimestamp", internal.IndexFuncs{
|
||||
EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
|
||||
return fields.Hash, nil
|
||||
},
|
||||
DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
|
||||
e.Hash = key
|
||||
return e, nil
|
||||
},
|
||||
EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
|
||||
b := make([]byte, 8)
|
||||
binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp))
|
||||
return b, nil
|
||||
},
|
||||
DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
|
||||
e.AccessTimestamp = int64(binary.BigEndian.Uint64(value))
|
||||
return e, nil
|
||||
},
|
||||
})
|
||||
db.gcIndex, err = idb.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", internal.IndexFuncs{
|
||||
EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
|
||||
b := make([]byte, 16, 16+len(fields.Hash))
|
||||
binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp))
|
||||
binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp))
|
||||
key = append(b, fields.Hash...)
|
||||
return key, nil
|
||||
},
|
||||
DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
|
||||
e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8]))
|
||||
e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16]))
|
||||
e.Hash = key[16:]
|
||||
return e, nil
|
||||
},
|
||||
EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
|
||||
return nil, nil
|
||||
},
|
||||
DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
|
||||
return e, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (db *DB) Put(_ context.Context, ch storage.Chunk) (err error) {
|
||||
return db.retrievalIndex.Put(internal.IndexItem{
|
||||
Hash: ch.Address(),
|
||||
Data: ch.Data(),
|
||||
StoreTimestamp: time.Now().UTC().UnixNano(),
|
||||
})
|
||||
}
|
||||
|
||||
func (db *DB) Get(_ context.Context, ref storage.Address) (c storage.Chunk, err error) {
|
||||
batch := new(leveldb.Batch)
|
||||
|
||||
item, err := db.retrievalIndex.Get(internal.IndexItem{
|
||||
Hash: ref,
|
||||
})
|
||||
if err != nil {
|
||||
if err == leveldb.ErrNotFound {
|
||||
return nil, storage.ErrChunkNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
accessItem, err := db.accessIndex.Get(internal.IndexItem{
|
||||
Hash: ref,
|
||||
})
|
||||
switch err {
|
||||
case nil:
|
||||
err = db.gcIndex.DeleteInBatch(batch, internal.IndexItem{
|
||||
Hash: item.Hash,
|
||||
StoreTimestamp: accessItem.AccessTimestamp,
|
||||
AccessTimestamp: item.StoreTimestamp,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case leveldb.ErrNotFound:
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
accessTimestamp := time.Now().UTC().UnixNano()
|
||||
|
||||
err = db.accessIndex.PutInBatch(batch, internal.IndexItem{
|
||||
Hash: ref,
|
||||
AccessTimestamp: accessTimestamp,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.gcIndex.PutInBatch(batch, internal.IndexItem{
|
||||
Hash: item.Hash,
|
||||
AccessTimestamp: accessTimestamp,
|
||||
StoreTimestamp: item.StoreTimestamp,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = db.db.WriteBatch(batch)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return storage.NewChunk(item.Hash, item.Data), nil
|
||||
}
|
||||
|
||||
func (db *DB) CollectGarbage() (err error) {
|
||||
const maxTrashSize = 100
|
||||
maxRounds := 10 // adbitrary number, needs to be calculated
|
||||
|
||||
for roundCount := 0; roundCount < maxRounds; roundCount++ {
|
||||
var garbageCount int
|
||||
trash := new(leveldb.Batch)
|
||||
err = db.gcIndex.IterateAll(func(item internal.IndexItem) (stop bool, err error) {
|
||||
err = db.retrievalIndex.DeleteInBatch(trash, item)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = db.accessIndex.DeleteInBatch(trash, item)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
err = db.gcIndex.DeleteInBatch(trash, item)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
garbageCount++
|
||||
if garbageCount >= maxTrashSize {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if garbageCount == 0 {
|
||||
return nil
|
||||
}
|
||||
err = db.db.WriteBatch(trash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) GetSchema() (name string, err error) {
|
||||
name, err = db.schemaName.Get()
|
||||
if err == leveldb.ErrNotFound {
|
||||
return "", nil
|
||||
}
|
||||
return name, err
|
||||
}
|
||||
|
||||
func (db *DB) PutSchema(name string) (err error) {
|
||||
return db.schemaName.Put(name)
|
||||
}
|
||||
|
||||
func (db *DB) Close() {
|
||||
db.db.Close()
|
||||
}
|
||||
|
||||
func Example_dbstore() {
|
||||
dir, err := ioutil.TempDir("", "ephemeral")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
db, err := New(dir)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ch := storage.GenerateRandomChunk(1024)
|
||||
err = db.Put(context.Background(), ch)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := db.Get(context.Background(), ch.Address())
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Println(bytes.Equal(got.Data(), ch.Data()))
|
||||
|
||||
//Output: true
|
||||
}
|
||||
Loading…
Reference in a new issue