forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-19 18:32:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

swarm/shed: remove internal package and add comments

Browse source
This commit is contained in:
Janos Guljas 2018-11-14 15:38:21 +01:00
parent 6d6afef911
commit a919a3214d
15 changed files with 543 additions and 600 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
swarm/shed/internal/db.go → swarm/shed/db.go
View file

@ -14,7 +14,13 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
// Package shed provides a simple abstraction components to compose
// more complex operations on storage data organized in fields and indexes.
//
// Only type which holds logical information about swarm storage chunks data
// and metadata is IndexItem. This part is not generalized mostly for
// performance reasons.
package shed
import (
"github.com/ethereum/go-ethereum/metrics"
@ -23,21 +29,33 @@ import (
"github.com/syndtr/goleveldb/leveldb/opt"
)
// The limit for LevelDB OpenFilesCacheCapacity.
const openFileLimit = 128
// DB provides abstractions over LevelDB in order to
// implement complex structures using fields and ordered indexes.
// It provides a schema functionality to store fields and indexes
// information about naming and types.
type DB struct {
ldb *leveldb.DB
}
// NewDB constructs a new DB and validates the schema
// if it exists in database on the given path.
func NewDB(path string) (db *DB, err error) {
ldb, err := leveldb.OpenFile(path, &opt.Options{OpenFilesCacheCapacity: openFileLimit})
ldb, err := leveldb.OpenFile(path, &opt.Options{
OpenFilesCacheCapacity: openFileLimit,
})
if err != nil {
return nil, err
}
db = &DB{ldb: ldb}
db = &DB{
ldb: ldb,
}
if _, err = db.getSchema(); err != nil {
if err == leveldb.ErrNotFound {
// save schema with initialized default fields
if err = db.putSchema(schema{
Fields: make(map[string]fieldSpec),
Indexes: make(map[byte]indexSpec),
@ -51,34 +69,42 @@ func NewDB(path string) (db *DB, err error) {
return db, nil
}
// Put wraps LevelDB Put method to increment metrics counter.
func (db *DB) Put(key []byte, value []byte) (err error) {
metrics.GetOrRegisterCounter("DB.put", nil).Inc(1)
return db.ldb.Put(key, value, nil)
}
// Get wraps LevelDB Get method to increment metrics counter.
func (db *DB) Get(key []byte) (value []byte, err error) {
metrics.GetOrRegisterCounter("DB.get", nil).Inc(1)
return db.ldb.Get(key, nil)
}
// Delete wraps LevelDB Delete method to increment metrics counter.
func (db *DB) Delete(key []byte) error {
metrics.GetOrRegisterCounter("DB.delete", nil).Inc(1)
return db.ldb.Delete(key, nil)
}
// NewIterator wraps LevelDB NewIterator method to increment metrics counter.
func (db *DB) NewIterator() iterator.Iterator {
metrics.GetOrRegisterCounter("DB.newiterator", nil).Inc(1)
return db.ldb.NewIterator(nil, nil)
}
// WriteBatch wraps LevelDB Write method to increment metrics counter.
func (db *DB) WriteBatch(batch *leveldb.Batch) error {
metrics.GetOrRegisterCounter("DB.write", nil).Inc(1)
return db.ldb.Write(batch, nil)
}
// Close closes LevelDB database.
func (db *DB) Close() (err error) {
return db.ldb.Close()
}

9
swarm/shed/internal/db_test.go → swarm/shed/db_test.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"io/ioutil"
@ -22,6 +22,8 @@ import (
"testing"
)
// TestNewDB constructs a new DB
// and validates if the schema is initialized properly.
func TestNewDB(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
@ -44,6 +46,8 @@ func TestNewDB(t *testing.T) {
}
}
// TestDB_persistence creates one DB, saves a field and closes that DB.
// Then, it constructs another DB and trues to retrieve the saved value.
func TestDB_persistence(t *testing.T) {
dir, err := ioutil.TempDir("", "shed-test-persistence")
if err != nil {
@ -86,6 +90,9 @@ func TestDB_persistence(t *testing.T) {
}
}
// newTestDB is a helper function that constructs a
// temporary database and returns a cleanup function that must
// be called to remove the data.
func newTestDB(t *testing.T) (db *DB, cleanupFunc func()) {
t.Helper()

325
swarm/shed/example_store_test.go Normal file
View file

@ -0,0 +1,325 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed_test
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"io/ioutil"
"log"
"os"
"time"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
// Store holds fields and indexes (including their encoding functions)
// and defines operations on them by composing data from them.
// It implements storage.ChunkStore interface.
// It is just an example without any support for parallel operations
// or real world implementation.
type Store struct {
db *shed.DB
// fields and indexes
schemaName shed.StringField
sizeCounter shed.Uint64Field
accessCounter shed.Uint64Field
retrievalIndex shed.Index
accessIndex shed.Index
gcIndex shed.Index
}
// New returns new Store. All fields and indexes are initialized
// and possible conflicts with schema from existing database is checked
// automatically.
func New(path string) (s *Store, err error) {
db, err := shed.NewDB(path)
if err != nil {
return nil, err
}
s = &Store{
db: db,
}
// Identify current storage schema by arbitrary name.
s.schemaName, err = db.NewStringField("schema-name")
if err != nil {
return nil, err
}
// Global ever incrementing index of chunk accesses.
s.accessCounter, err = db.NewUint64Field("access-counter")
if err != nil {
return nil, err
}
// Index storing actual chunk address, data and store timestamp.
s.retrievalIndex, err = db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{
EncodeKey: func(fields shed.IndexItem) (key []byte, err error) {
return fields.Address, nil
},
DecodeKey: func(key []byte) (e shed.IndexItem, err error) {
e.Address = key
return e, nil
},
EncodeValue: func(fields shed.IndexItem) (value []byte, err error) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp))
value = append(b, fields.Data...)
return value, nil
},
DecodeValue: func(value []byte) (e shed.IndexItem, err error) {
e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8]))
e.Data = value[8:]
return e, nil
},
})
// Index storing access timestamp for a particular address.
// It is needed in order to update gc index keys for iteration order.
s.accessIndex, err = db.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{
EncodeKey: func(fields shed.IndexItem) (key []byte, err error) {
return fields.Address, nil
},
DecodeKey: func(key []byte) (e shed.IndexItem, err error) {
e.Address = key
return e, nil
},
EncodeValue: func(fields shed.IndexItem) (value []byte, err error) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp))
return b, nil
},
DecodeValue: func(value []byte) (e shed.IndexItem, err error) {
e.AccessTimestamp = int64(binary.BigEndian.Uint64(value))
return e, nil
},
})
// Index with keys ordered by access timestamp for garbage collection prioritization.
s.gcIndex, err = db.NewIndex("AccessTimestamp|StoredTimestamp|Address->nil", shed.IndexFuncs{
EncodeKey: func(fields shed.IndexItem) (key []byte, err error) {
b := make([]byte, 16, 16+len(fields.Address))
binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp))
binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp))
key = append(b, fields.Address...)
return key, nil
},
DecodeKey: func(key []byte) (e shed.IndexItem, err error) {
e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8]))
e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16]))
e.Address = key[16:]
return e, nil
},
EncodeValue: func(fields shed.IndexItem) (value []byte, err error) {
return nil, nil
},
DecodeValue: func(value []byte) (e shed.IndexItem, err error) {
return e, nil
},
})
if err != nil {
return nil, err
}
return s, nil
}
// Put stores the chunk and sets it store timestamp.
func (s *Store) Put(_ context.Context, ch storage.Chunk) (err error) {
return s.retrievalIndex.Put(shed.IndexItem{
Address: ch.Address(),
Data: ch.Data(),
StoreTimestamp: time.Now().UTC().UnixNano(),
})
}
// Get retrieves a chunk with the provided address.
// It updates access and gc indexes by removing the previous
// items from them and adding new items as keys of index entries
// are changed.
func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, err error) {
batch := new(leveldb.Batch)
// Get the chunk data and storage timestamp.
item, err := s.retrievalIndex.Get(shed.IndexItem{
Address: addr,
})
if err != nil {
if err == leveldb.ErrNotFound {
return nil, storage.ErrChunkNotFound
}
return nil, err
}
// Get the chunk access timestamp.
accessItem, err := s.accessIndex.Get(shed.IndexItem{
Address: addr,
})
switch err {
case nil:
// Remove gc index entry if access timestamp is found.
err = s.gcIndex.DeleteInBatch(batch, shed.IndexItem{
Address: item.Address,
StoreTimestamp: accessItem.AccessTimestamp,
AccessTimestamp: item.StoreTimestamp,
})
if err != nil {
return nil, err
}
case leveldb.ErrNotFound:
// Access timestamp is not found. Do not do anything.
// This is the firs get request.
default:
return nil, err
}
// Specify new access timestamp
accessTimestamp := time.Now().UTC().UnixNano()
// Put new access timestamp in access index.
err = s.accessIndex.PutInBatch(batch, shed.IndexItem{
Address: addr,
AccessTimestamp: accessTimestamp,
})
if err != nil {
return nil, err
}
// Put new access timestamp in gc index.
err = s.gcIndex.PutInBatch(batch, shed.IndexItem{
Address: item.Address,
AccessTimestamp: accessTimestamp,
StoreTimestamp: item.StoreTimestamp,
})
if err != nil {
return nil, err
}
// Increment access counter.
// Currently this information is not used anywhere.
_, err = s.accessCounter.IncInBatch(batch)
if err != nil {
return nil, err
}
// Write the batch.
err = s.db.WriteBatch(batch)
if err != nil {
return nil, err
}
// Return the chunk.
return storage.NewChunk(item.Address, item.Data), nil
}
// CollectGarbage is an example of index iteration.
// It provides no reliable garbage collection functionality.
func (s *Store) CollectGarbage() (err error) {
const maxTrashSize = 100
maxRounds := 10 // arbitrary number, needs to be calculated
// Run a few gc rounds.
for roundCount := 0; roundCount < maxRounds; roundCount++ {
var garbageCount int
// New batch for a new cg round.
trash := new(leveldb.Batch)
// Iterate through all index items and break when needed.
err = s.gcIndex.IterateAll(func(item shed.IndexItem) (stop bool, err error) {
// Remove the chunk.
err = s.retrievalIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
// Remove the element in gc index.
err = s.gcIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
// Remove the relation in access index.
err = s.accessIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
garbageCount++
if garbageCount >= maxTrashSize {
return true, nil
}
return false, nil
})
if err != nil {
return err
}
if garbageCount == 0 {
return nil
}
err = s.db.WriteBatch(trash)
if err != nil {
return err
}
}
return nil
}
// GetSchema is an example of retrieveing the most simple
// string from a database field.
func (s *Store) GetSchema() (name string, err error) {
name, err = s.schemaName.Get()
if err == leveldb.ErrNotFound {
return "", nil
}
return name, err
}
// GetSchema is an example of storing the most simple
// string in a database field.
func (s *Store) PutSchema(name string) (err error) {
return s.schemaName.Put(name)
}
// Close closes the underlying database.
func (s *Store) Close() {
s.db.Close()
}
// Example_store constructs a simple storage implementation using shed package.
func Example_store() {
dir, err := ioutil.TempDir("", "ephemeral")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(dir)
s, err := New(dir)
if err != nil {
log.Fatal(err)
}
ch := storage.GenerateRandomChunk(1024)
err = s.Put(context.Background(), ch)
if err != nil {
log.Fatal(err)
}
got, err := s.Get(context.Background(), ch.Address())
if err != nil {
log.Fatal(err)
}
fmt.Println(bytes.Equal(got.Data(), ch.Data()))
//Output: true
}

14
swarm/shed/internal/field_json.go → swarm/shed/field_json.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"encoding/json"
@ -22,11 +22,15 @@ import (
"github.com/syndtr/goleveldb/leveldb"
)
// JSONField is a helper to store complex structure by
// encoding it in JSON format.
type JSONField struct {
db *DB
key []byte
}
// NewJSONField returns a new JSONField.
// It validates its name and type against the database schema.
func (db *DB) NewJSONField(name string) (f JSONField, err error) {
key, err := db.schemaFieldKey(name, "json")
if err != nil {
@ -38,18 +42,17 @@ func (db *DB) NewJSONField(name string) (f JSONField, err error) {
}, nil
}
// Unmarshal unmarshals data fromt he database to a provided val.
// If the data is not found leveldb.ErrNotFound is returned.
func (f JSONField) Unmarshal(val interface{}) (err error) {
b, err := f.db.Get(f.key)
if err != nil {
// Q: should we ignore not found
// if err == leveldb.ErrNotFound {
// return nil
// }
return err
}
return json.Unmarshal(b, val)
}
// Put marshals provided val and saves it to the database.
func (f JSONField) Put(val interface{}) (err error) {
b, err := json.Marshal(val)
if err != nil {
@ -58,6 +61,7 @@ func (f JSONField) Put(val interface{}) (err error) {
return f.db.Put(f.key, b)
}
// PutInBatch marshals provided val and puts it into the batch.
func (f JSONField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) {
b, err := json.Marshal(val)
if err != nil {

4
swarm/shed/internal/field_json_test.go → swarm/shed/field_json_test.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"testing"
@ -22,6 +22,8 @@ import (
"github.com/syndtr/goleveldb/leveldb"
)
// TestJSONField validates put and unmarshal operations
// of the JSONField.
func TestJSONField(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()

12
swarm/shed/internal/field_string.go → swarm/shed/field_string.go
View file

@ -14,17 +14,21 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"github.com/syndtr/goleveldb/leveldb"
)
// StringField is the most simple field implementation
// that stores an arbitrary string under a specific LevelDB key.
type StringField struct {
db *DB
key []byte
}
// NewStringField retruns a new Instance fo StringField.
// It validates its name and type against the database schema.
func (db *DB) NewStringField(name string) (f StringField, err error) {
key, err := db.schemaFieldKey(name, "string")
if err != nil {
@ -36,6 +40,9 @@ func (db *DB) NewStringField(name string) (f StringField, err error) {
}, nil
}
// Get returns a string value from database.
// If the value is not found, an empty string is returned
// an no error.
func (f StringField) Get() (val string, err error) {
b, err := f.db.Get(f.key)
if err != nil {
@ -47,10 +54,13 @@ func (f StringField) Get() (val string, err error) {
return string(b), nil
}
// Put stores a string in the database.
func (f StringField) Put(val string) (err error) {
return f.db.Put(f.key, []byte(val))
}
// PutInBatch stores a string in a batch that can be
// saved later in database.
func (f StringField) PutInBatch(batch *leveldb.Batch, val string) {
batch.Put(f.key, []byte(val))
}

4
swarm/shed/internal/field_string_test.go → swarm/shed/field_string_test.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"testing"
@ -22,6 +22,8 @@ import (
"github.com/syndtr/goleveldb/leveldb"
)
// TestStringField validates put and get operations
// of the StringField.
func TestStringField(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()

19
swarm/shed/internal/field_uint64.go → swarm/shed/field_uint64.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"encoding/binary"
@ -22,11 +22,15 @@ import (
"github.com/syndtr/goleveldb/leveldb"
)
// Uint64Field provides a way to have a simple counter in the database.
// It transparently encodes uint64 type value to bytes.
type Uint64Field struct {
db *DB
key []byte
}
// NewUint64Field returns a new Uint64Field.
// It validates its name and type against the database schema.
func (db *DB) NewUint64Field(name string) (f Uint64Field, err error) {
key, err := db.schemaFieldKey(name, "uint64")
if err != nil {
@ -38,6 +42,9 @@ func (db *DB) NewUint64Field(name string) (f Uint64Field, err error) {
}, nil
}
// Get retrieves a uint64 value from the database.
// If the value is not found in the database a 0 value
// is returned and no error.
func (f Uint64Field) Get() (val uint64, err error) {
b, err := f.db.Get(f.key)
if err != nil {
@ -49,14 +56,19 @@ func (f Uint64Field) Get() (val uint64, err error) {
return binary.BigEndian.Uint64(b), nil
}
// Put encodes uin64 value and stores it in the database.
func (f Uint64Field) Put(val uint64) (err error) {
return f.db.Put(f.key, encodeUint64(val))
}
// PutInBatch stores a uint64 value in a batch
// that can be saved later in the database.
func (f Uint64Field) PutInBatch(batch *leveldb.Batch, val uint64) {
batch.Put(f.key, encodeUint64(val))
}
// Inc increments a uint64 value in the database.
// This operation is not goroutine save.
func (f Uint64Field) Inc() (val uint64, err error) {
val, err = f.Get()
if err != nil {
@ -70,6 +82,9 @@ func (f Uint64Field) Inc() (val uint64, err error) {
return val, f.Put(val)
}
// IncInBatch increments a uint64 value in the batch
// by retreiving a value from the database, not the same batch.
// This operation is not goroutine save.
func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) {
val, err = f.Get()
if err != nil {
@ -84,6 +99,8 @@ func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) {
return val, nil
}
// encode transforms uint64 to 8 byte long
// slice in big endian encoding.
func encodeUint64(val uint64) (b []byte) {
b = make([]byte, 8)
binary.BigEndian.PutUint64(b, val)

10
swarm/shed/internal/field_uint64_test.go → swarm/shed/field_uint64_test.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"testing"
@ -22,6 +22,8 @@ import (
"github.com/syndtr/goleveldb/leveldb"
)
// TestUint64Field validates put and get operations
// of the Uint64Field.
func TestUint64Field(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
@ -36,7 +38,7 @@ func TestUint64Field(t *testing.T) {
if err != nil {
t.Fatal(err)
}
var want uint64 = 0
var want uint64
if got != want {
t.Errorf("got uint64 %v, want %v", got, want)
}
@ -107,6 +109,8 @@ func TestUint64Field(t *testing.T) {
})
}
// TestUint64Field_Inc validates Inc operation
// of the Uint64Field.
func TestUint64Field_Inc(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
@ -135,6 +139,8 @@ func TestUint64Field_Inc(t *testing.T) {
}
}
// TestUint64Field_IncInBatch validates IncInBatch operation
// of the Uint64Field.
func TestUint64Field_IncInBatch(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()

62
swarm/shed/internal/index.go → swarm/shed/index.go
View file

@ -14,22 +14,37 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"github.com/syndtr/goleveldb/leveldb"
)
// IndexItem holds fields relevant to Swarm Chunk data and metadata.
// All information required for swarm storage and operations
// on that storage must be defined here.
// This structure is logically connected to swarm storage,
// the only part of this package that is not generalized,
// mostly for performance reasons.
//
// IndexItem is a type that is used for retrieving, storing and encoding
// chunk data and metadata. It is passed as an argument to Index encoding
// functions, get function and put function.
// But it is also returned with additional data from get function call
// and as the argument in iterator function definition.
type IndexItem struct {
Hash []byte
Address []byte
Data []byte
AccessTimestamp int64
StoreTimestamp int64
}
// Join is a helper method to construct a new
// IndexItem by filling up fields with default values
// of a particular IndexItem with values from another one.
func (i IndexItem) Join(i2 IndexItem) (new IndexItem) {
if i.Hash == nil {
i.Hash = i2.Hash
if i.Address == nil {
i.Address = i2.Address
}
if i.Data == nil {
i.Data = i2.Data
@ -43,6 +58,12 @@ func (i IndexItem) Join(i2 IndexItem) (new IndexItem) {
return i
}
// Index represents a set of LevelDB key value pairs that have common
// prefix. It holds functions for encoding and decoding keys and values
// to provide transparent actions on saved data which inclide:
// - getting a particular IndexItem
// - saving a particular IndexItem
// - iterating over a sorted LevelDB keys
type Index struct {
db *DB
prefix []byte
@ -52,6 +73,8 @@ type Index struct {
decodeValueFunc func(value []byte) (e IndexItem, err error)
}
// IndexFuncs structure defines functions for encoding and decoding
// LevelDB keys and values for a specific index.
type IndexFuncs struct {
EncodeKey func(fields IndexItem) (key []byte, err error)
DecodeKey func(key []byte) (e IndexItem, err error)
@ -59,8 +82,11 @@ type IndexFuncs struct {
DecodeValue func(value []byte) (e IndexItem, err error)
}
// NewIndex returns a new Index instance with defined name and
// encoding functions. The name must be unique and will be validated
// on database schema for a key prefix byte.
func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) {
id, err := db.schemaIndexID(name)
id, err := db.schemaIndexPrefix(name)
if err != nil {
return f, err
}
@ -83,6 +109,9 @@ func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) {
}, nil
}
// Get accepts key fields represented as IndexItem to retrieve a
// value from the index and return maximum available information
// from the index represented as another IndexItem.
func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) {
key, err := f.encodeKeyFunc(keyFields)
if err != nil {
@ -99,6 +128,8 @@ func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) {
return out.Join(keyFields), nil
}
// Put accepts IndexItem to encode information from it
// and save it to the database.
func (f Index) Put(i IndexItem) (err error) {
key, err := f.encodeKeyFunc(i)
if err != nil {
@ -111,6 +142,9 @@ func (f Index) Put(i IndexItem) (err error) {
return f.db.Put(key, value)
}
// PutInBatch is the same as Put method, but it just
// saves the key/value pair to the batch instead
// directly to the database.
func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) {
key, err := f.encodeKeyFunc(i)
if err != nil {
@ -124,6 +158,8 @@ func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) {
return nil
}
// Delete accepts IndexItem to remove a key/value pair
// form the database based on its fields.
func (f Index) Delete(keyFields IndexItem) (err error) {
key, err := f.encodeKeyFunc(keyFields)
if err != nil {
@ -132,6 +168,8 @@ func (f Index) Delete(keyFields IndexItem) (err error) {
return f.db.Delete(key)
}
// DeleteInBatch is the same as Delete just the operation
// is performed on the batch instead on the database.
func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error) {
key, err := f.encodeKeyFunc(keyFields)
if err != nil {
@ -141,9 +179,15 @@ func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err err
return nil
}
type IterFunc func(item IndexItem) (stop bool, err error)
// IndexIterFunc is a callback on every IndexItem that is decoded
// by iterating on an Index keys.
// By returning a true for stop variable, iteration will
// stop, and by returning the error, that error will be
// propagated to the called iterator method on Index.
type IndexIterFunc func(item IndexItem) (stop bool, err error)
func (f Index) IterateAll(fn IterFunc) (err error) {
// IterateAll iterates over all keys of the Index.
func (f Index) IterateAll(fn IndexIterFunc) (err error) {
it := f.db.NewIterator()
defer it.Release()
@ -171,7 +215,9 @@ func (f Index) IterateAll(fn IterFunc) (err error) {
return it.Error()
}
func (f Index) IterateFrom(start IndexItem, fn IterFunc) (err error) {
// IterateFrom iterates over Index keys starting from the key
// encoded from the provided IndexItem.
func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) {
startKey, err := f.encodeKeyFunc(start)
if err != nil {
return err

76
swarm/shed/internal/index_test.go → swarm/shed/index_test.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"bytes"
@ -27,12 +27,13 @@ import (
"github.com/syndtr/goleveldb/leveldb"
)
// Index functions for the index that is used in tests in this file.
var retrievalIndexFuncs = IndexFuncs{
EncodeKey: func(fields IndexItem) (key []byte, err error) {
return fields.Hash, nil
return fields.Address, nil
},
DecodeKey: func(key []byte) (e IndexItem, err error) {
e.Hash = key
e.Address = key
return e, nil
},
EncodeValue: func(fields IndexItem) (value []byte, err error) {
@ -48,6 +49,7 @@ var retrievalIndexFuncs = IndexFuncs{
},
}
// TestIndex validates put, get and delete functions of the index.
func TestIndex(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
@ -59,7 +61,7 @@ func TestIndex(t *testing.T) {
t.Run("put", func(t *testing.T) {
want := IndexItem{
Hash: []byte("put-hash"),
Address: []byte("put-hash"),
Data: []byte("DATA"),
StoreTimestamp: time.Now().UTC().UnixNano(),
}
@ -69,7 +71,7 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
got, err := index.Get(IndexItem{
Hash: want.Hash,
Address: want.Address,
})
if err != nil {
t.Fatal(err)
@ -78,7 +80,7 @@ func TestIndex(t *testing.T) {
t.Run("overwrite", func(t *testing.T) {
want := IndexItem{
Hash: []byte("put-hash"),
Address: []byte("put-hash"),
Data: []byte("New DATA"),
StoreTimestamp: time.Now().UTC().UnixNano(),
}
@ -88,7 +90,7 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
got, err := index.Get(IndexItem{
Hash: want.Hash,
Address: want.Address,
})
if err != nil {
t.Fatal(err)
@ -99,7 +101,7 @@ func TestIndex(t *testing.T) {
t.Run("put in batch", func(t *testing.T) {
want := IndexItem{
Hash: []byte("put-in-batch-hash"),
Address: []byte("put-in-batch-hash"),
Data: []byte("DATA"),
StoreTimestamp: time.Now().UTC().UnixNano(),
}
@ -111,7 +113,7 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
got, err := index.Get(IndexItem{
Hash: want.Hash,
Address: want.Address,
})
if err != nil {
t.Fatal(err)
@ -120,7 +122,7 @@ func TestIndex(t *testing.T) {
t.Run("overwrite", func(t *testing.T) {
want := IndexItem{
Hash: []byte("put-in-batch-hash"),
Address: []byte("put-in-batch-hash"),
Data: []byte("New DATA"),
StoreTimestamp: time.Now().UTC().UnixNano(),
}
@ -132,7 +134,7 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
got, err := index.Get(IndexItem{
Hash: want.Hash,
Address: want.Address,
})
if err != nil {
t.Fatal(err)
@ -143,7 +145,7 @@ func TestIndex(t *testing.T) {
t.Run("delete", func(t *testing.T) {
want := IndexItem{
Hash: []byte("delete-hash"),
Address: []byte("delete-hash"),
Data: []byte("DATA"),
StoreTimestamp: time.Now().UTC().UnixNano(),
}
@ -153,7 +155,7 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
got, err := index.Get(IndexItem{
Hash: want.Hash,
Address: want.Address,
})
if err != nil {
t.Fatal(err)
@ -161,14 +163,14 @@ func TestIndex(t *testing.T) {
checkIndexItem(t, got, want)
err = index.Delete(IndexItem{
Hash: want.Hash,
Address: want.Address,
})
if err != nil {
t.Fatal(err)
}
got, err = index.Get(IndexItem{
Hash: want.Hash,
Address: want.Address,
})
if err != leveldb.ErrNotFound {
t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
@ -177,7 +179,7 @@ func TestIndex(t *testing.T) {
t.Run("delete in batch", func(t *testing.T) {
want := IndexItem{
Hash: []byte("delete-in-batch-hash"),
Address: []byte("delete-in-batch-hash"),
Data: []byte("DATA"),
StoreTimestamp: time.Now().UTC().UnixNano(),
}
@ -187,7 +189,7 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
got, err := index.Get(IndexItem{
Hash: want.Hash,
Address: want.Address,
})
if err != nil {
t.Fatal(err)
@ -196,7 +198,7 @@ func TestIndex(t *testing.T) {
batch := new(leveldb.Batch)
index.DeleteInBatch(batch, IndexItem{
Hash: want.Hash,
Address: want.Address,
})
err = db.WriteBatch(batch)
if err != nil {
@ -204,7 +206,7 @@ func TestIndex(t *testing.T) {
}
got, err = index.Get(IndexItem{
Hash: want.Hash,
Address: want.Address,
})
if err != leveldb.ErrNotFound {
t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
@ -212,6 +214,7 @@ func TestIndex(t *testing.T) {
})
}
// TestIndex_iterate validates index iterator functions for correctness.
func TestIndex_iterate(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
@ -223,24 +226,24 @@ func TestIndex_iterate(t *testing.T) {
items := []IndexItem{
{
Hash: []byte("iterate-hash-01"),
Data: []byte("data80"),
Address: []byte("iterate-hash-01"),
Data: []byte("data80"),
},
{
Hash: []byte("iterate-hash-03"),
Data: []byte("data22"),
Address: []byte("iterate-hash-03"),
Data: []byte("data22"),
},
{
Hash: []byte("iterate-hash-05"),
Data: []byte("data41"),
Address: []byte("iterate-hash-05"),
Data: []byte("data41"),
},
{
Hash: []byte("iterate-hash-02"),
Data: []byte("data84"),
Address: []byte("iterate-hash-02"),
Data: []byte("data84"),
},
{
Hash: []byte("iterate-hash-06"),
Data: []byte("data1"),
Address: []byte("iterate-hash-06"),
Data: []byte("data1"),
},
}
batch := new(leveldb.Batch)
@ -252,8 +255,8 @@ func TestIndex_iterate(t *testing.T) {
t.Fatal(err)
}
item04 := IndexItem{
Hash: []byte("iterate-hash-04"),
Data: []byte("data0"),
Address: []byte("iterate-hash-04"),
Data: []byte("data0"),
}
err = index.Put(item04)
if err != nil {
@ -262,7 +265,7 @@ func TestIndex_iterate(t *testing.T) {
items = append(items, item04)
sort.SliceStable(items, func(i, j int) bool {
return bytes.Compare(items[i].Hash, items[j].Hash) < 0
return bytes.Compare(items[i].Address, items[j].Address) < 0
})
t.Run("all", func(t *testing.T) {
@ -331,8 +334,8 @@ func TestIndex_iterate(t *testing.T) {
}
secondIndexItem := IndexItem{
Hash: []byte("iterate-hash-10"),
Data: []byte("data-second"),
Address: []byte("iterate-hash-10"),
Data: []byte("data-second"),
}
err = secondIndex.Put(secondIndexItem)
if err != nil {
@ -368,11 +371,12 @@ func TestIndex_iterate(t *testing.T) {
})
}
// checkIndexItem is a test helper function that compares if two Index items are the same.
func checkIndexItem(t *testing.T, got, want IndexItem) {
t.Helper()
if !bytes.Equal(got.Hash, want.Hash) {
t.Errorf("got hash %q, expected %q", string(got.Hash), string(want.Hash))
if !bytes.Equal(got.Address, want.Address) {
t.Errorf("got hash %q, expected %q", string(got.Address), string(want.Address))
}
if !bytes.Equal(got.Data, want.Data) {
t.Errorf("got data %q, expected %q", string(got.Data), string(want.Data))

280
swarm/shed/internal/example_dbstore_test.go
View file

@ -1,280 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal_test
import (
"bytes"
"context"
"encoding/binary"
"fmt"
"io/ioutil"
"log"
"os"
"time"
"github.com/ethereum/go-ethereum/swarm/shed/internal"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
// DB is just an example for composing indexes.
type DB struct {
db *internal.DB
// fields and indexes
schemaName internal.StringField
sizeCounter internal.Uint64Field
accessCounter internal.Uint64Field
retrievalIndex internal.Index
accessIndex internal.Index
gcIndex internal.Index
}
func New(path string) (db *DB, err error) {
idb, err := internal.NewDB(path)
if err != nil {
return nil, err
}
db = &DB{
db: idb,
}
db.schemaName, err = idb.NewStringField("schema-name")
if err != nil {
return nil, err
}
db.sizeCounter, err = idb.NewUint64Field("size-counter")
if err != nil {
return nil, err
}
db.accessCounter, err = idb.NewUint64Field("access-counter")
if err != nil {
return nil, err
}
db.retrievalIndex, err = idb.NewIndex("Hash->StoreTimestamp|Data", internal.IndexFuncs{
EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
return fields.Hash, nil
},
DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
e.Hash = key
return e, nil
},
EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp))
value = append(b, fields.Data...)
return value, nil
},
DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8]))
e.Data = value[8:]
return e, nil
},
})
db.accessIndex, err = idb.NewIndex("Hash->AccessTimestamp", internal.IndexFuncs{
EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
return fields.Hash, nil
},
DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
e.Hash = key
return e, nil
},
EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp))
return b, nil
},
DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
e.AccessTimestamp = int64(binary.BigEndian.Uint64(value))
return e, nil
},
})
db.gcIndex, err = idb.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", internal.IndexFuncs{
EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
b := make([]byte, 16, 16+len(fields.Hash))
binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp))
binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp))
key = append(b, fields.Hash...)
return key, nil
},
DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8]))
e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16]))
e.Hash = key[16:]
return e, nil
},
EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
return nil, nil
},
DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
return e, nil
},
})
if err != nil {
return nil, err
}
return db, nil
}
func (db *DB) Put(_ context.Context, ch storage.Chunk) (err error) {
return db.retrievalIndex.Put(internal.IndexItem{
Hash: ch.Address(),
Data: ch.Data(),
StoreTimestamp: time.Now().UTC().UnixNano(),
})
}
func (db *DB) Get(_ context.Context, ref storage.Address) (c storage.Chunk, err error) {
batch := new(leveldb.Batch)
item, err := db.retrievalIndex.Get(internal.IndexItem{
Hash: ref,
})
if err != nil {
if err == leveldb.ErrNotFound {
return nil, storage.ErrChunkNotFound
}
return nil, err
}
accessItem, err := db.accessIndex.Get(internal.IndexItem{
Hash: ref,
})
switch err {
case nil:
err = db.gcIndex.DeleteInBatch(batch, internal.IndexItem{
Hash: item.Hash,
StoreTimestamp: accessItem.AccessTimestamp,
AccessTimestamp: item.StoreTimestamp,
})
if err != nil {
return nil, err
}
case leveldb.ErrNotFound:
default:
return nil, err
}
accessTimestamp := time.Now().UTC().UnixNano()
err = db.accessIndex.PutInBatch(batch, internal.IndexItem{
Hash: ref,
AccessTimestamp: accessTimestamp,
})
if err != nil {
return nil, err
}
err = db.gcIndex.PutInBatch(batch, internal.IndexItem{
Hash: item.Hash,
AccessTimestamp: accessTimestamp,
StoreTimestamp: item.StoreTimestamp,
})
if err != nil {
return nil, err
}
err = db.db.WriteBatch(batch)
if err != nil {
return nil, err
}
return storage.NewChunk(item.Hash, item.Data), nil
}
func (db *DB) CollectGarbage() (err error) {
const maxTrashSize = 100
maxRounds := 10 // adbitrary number, needs to be calculated
for roundCount := 0; roundCount < maxRounds; roundCount++ {
var garbageCount int
trash := new(leveldb.Batch)
err = db.gcIndex.IterateAll(func(item internal.IndexItem) (stop bool, err error) {
err = db.retrievalIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
err = db.accessIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
err = db.gcIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
garbageCount++
if garbageCount >= maxTrashSize {
return true, nil
}
return false, nil
})
if err != nil {
return err
}
if garbageCount == 0 {
return nil
}
err = db.db.WriteBatch(trash)
if err != nil {
return err
}
}
return nil
}
func (db *DB) GetSchema() (name string, err error) {
name, err = db.schemaName.Get()
if err == leveldb.ErrNotFound {
return "", nil
}
return name, err
}
func (db *DB) PutSchema(name string) (err error) {
return db.schemaName.Put(name)
}
func (db *DB) Close() {
db.db.Close()
}
func Example_dbstore() {
dir, err := ioutil.TempDir("", "ephemeral")
if err != nil {
log.Fatal(err)
}
defer os.RemoveAll(dir)
db, err := New(dir)
if err != nil {
log.Fatal(err)
}
ch := storage.GenerateRandomChunk(1024)
err = db.Put(context.Background(), ch)
if err != nil {
log.Fatal(err)
}
got, err := db.Get(context.Background(), ch.Address())
if err != nil {
log.Fatal(err)
}
fmt.Println(bytes.Equal(got.Data(), ch.Data()))
//Output: true
}

33
swarm/shed/internal/schema.go → swarm/shed/schema.go
View file

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"encoding/json"
@ -23,24 +23,37 @@ import (
)
var (
keySchema = []byte{0}
keyPrefixFields byte = 1
keyPrefixIndexStart byte = 2 // Q: or maybe 7, to have more space for potential specific perfixes
// LevelDB key value for storing the schema.
keySchema = []byte{0}
// LevelDB key prefix for all field type.
// LevelDB keys will be constructed by appending name values to this prefix.
keyPrefixFields byte = 1
// LevelDB key prefix from which indexing keys start.
// Every index has its own key prefix and this value defines the first one.
keyPrefixIndexStart byte = 2 // Q: or maybe a higher number like 7, to have more space for potential specific perfixes
)
// schema is used to serialize known database structure information.
type schema struct {
Fields map[string]fieldSpec `json:"fields"`
Indexes map[byte]indexSpec `json:"indexes"`
Fields map[string]fieldSpec `json:"fields"` // keys are field names
Indexes map[byte]indexSpec `json:"indexes"` // keys are index prefix bytes
}
// fieldSpec holds information about a particular field.
// It does not need Name field as it is contained in the
// schema.Field map key.
type fieldSpec struct {
Type string `json:"type"`
}
// indxSpec holds information about a particular index.
// It does not contain index type, as indexes do not have type.
type indexSpec struct {
Name string `json:"name"`
}
// schemaFieldKey retrives the complete LevelDB key for
// a particular field form the schema definition.
func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) {
if name == "" {
return nil, errors.New("filed name can not be blank")
@ -73,7 +86,9 @@ func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) {
return append([]byte{keyPrefixFields}, []byte(name)...), nil
}
func (db *DB) schemaIndexID(name string) (id byte, err error) {
// schemaIndexID retrieves the complete LevelDB prefix for
// a particular index.
func (db *DB) schemaIndexPrefix(name string) (id byte, err error) {
if name == "" {
return 0, errors.New("index name can not be blank")
}
@ -97,6 +112,8 @@ func (db *DB) schemaIndexID(name string) (id byte, err error) {
return id, db.putSchema(s)
}
// getSchema retrieves the complete schema from
// the database.
func (db *DB) getSchema() (s schema, err error) {
b, err := db.Get(keySchema)
if err != nil {
@ -106,6 +123,8 @@ func (db *DB) getSchema() (s schema, err error) {
return s, err
}
// putSchema stores the complete schema to
// the database.
func (db *DB) putSchema(s schema) (err error) {
b, err := json.Marshal(s)
if err != nil {

16
swarm/shed/internal/schema_test.go → swarm/shed/schema_test.go
View file

@ -14,14 +14,15 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package internal
package shed
import (
"bytes"
"testing"
)
func TestSchema_schemaFieldKey(t *testing.T) {
// TestDB_schemaFieldKey validates correctness of schemaFieldKey.
func TestDB_schemaFieldKey(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
@ -86,17 +87,18 @@ func TestSchema_schemaFieldKey(t *testing.T) {
})
}
func TestSchema_schemaIndexID(t *testing.T) {
// TestDB_schemaIndexPrefix validates correctness of schemaIndexPrefix.
func TestDB_schemaIndexPrefix(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
t.Run("same name", func(t *testing.T) {
id1, err := db.schemaIndexID("test")
id1, err := db.schemaIndexPrefix("test")
if err != nil {
t.Fatal(err)
}
id2, err := db.schemaIndexID("test")
id2, err := db.schemaIndexPrefix("test")
if err != nil {
t.Fatal(err)
}
@ -107,12 +109,12 @@ func TestSchema_schemaIndexID(t *testing.T) {
})
t.Run("different names", func(t *testing.T) {
id1, err := db.schemaIndexID("test1")
id1, err := db.schemaIndexPrefix("test1")
if err != nil {
t.Fatal(err)
}
id2, err := db.schemaIndexID("test2")
id2, err := db.schemaIndexPrefix("test2")
if err != nil {
t.Fatal(err)
}

247
swarm/shed/shed.go
View file

@ -1,247 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"context"
"encoding/binary"
"time"
"github.com/ethereum/go-ethereum/swarm/shed/internal"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/syndtr/goleveldb/leveldb"
)
// DB is just an example for composing indexes.
type DB struct {
db *internal.DB
// fields and indexes
schemaName internal.StringField
sizeCounter internal.Uint64Field
accessCounter internal.Uint64Field
retrievalIndex internal.Index
accessIndex internal.Index
gcIndex internal.Index
}
func New(path string) (db *DB, err error) {
idb, err := internal.NewDB(path)
if err != nil {
return nil, err
}
db = &DB{
db: idb,
}
db.schemaName, err = idb.NewStringField("schema-name")
if err != nil {
return nil, err
}
db.sizeCounter, err = idb.NewUint64Field("size-counter")
if err != nil {
return nil, err
}
db.accessCounter, err = idb.NewUint64Field("access-counter")
if err != nil {
return nil, err
}
db.retrievalIndex, err = idb.NewIndex("Hash->StoreTimestamp|Data", internal.IndexFuncs{
EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
return fields.Hash, nil
},
DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
e.Hash = key
return e, nil
},
EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp))
value = append(b, fields.Data...)
return value, nil
},
DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8]))
e.Data = value[8:]
return e, nil
},
})
db.accessIndex, err = idb.NewIndex("Hash->AccessTimestamp", internal.IndexFuncs{
EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
return fields.Hash, nil
},
DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
e.Hash = key
return e, nil
},
EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp))
return b, nil
},
DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
e.AccessTimestamp = int64(binary.BigEndian.Uint64(value))
return e, nil
},
})
db.gcIndex, err = idb.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", internal.IndexFuncs{
EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
b := make([]byte, 16, 16+len(fields.Hash))
binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp))
binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp))
key = append(b, fields.Hash...)
return key, nil
},
DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8]))
e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16]))
e.Hash = key[16:]
return e, nil
},
EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
return nil, nil
},
DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
return e, nil
},
})
if err != nil {
return nil, err
}
return db, nil
}
func (db *DB) Put(_ context.Context, ch storage.Chunk) (err error) {
return db.retrievalIndex.Put(internal.IndexItem{
Hash: ch.Address(),
Data: ch.Data(),
StoreTimestamp: time.Now().UTC().UnixNano(),
})
}
func (db *DB) Get(_ context.Context, ref storage.Address) (c storage.Chunk, err error) {
batch := new(leveldb.Batch)
item, err := db.retrievalIndex.Get(internal.IndexItem{
Hash: ref,
})
if err != nil {
if err == leveldb.ErrNotFound {
return nil, storage.ErrChunkNotFound
}
return nil, err
}
accessItem, err := db.accessIndex.Get(internal.IndexItem{
Hash: ref,
})
switch err {
case nil:
err = db.gcIndex.DeleteInBatch(batch, internal.IndexItem{
Hash: item.Hash,
StoreTimestamp: accessItem.AccessTimestamp,
AccessTimestamp: item.StoreTimestamp,
})
if err != nil {
return nil, err
}
case leveldb.ErrNotFound:
default:
return nil, err
}
accessTimestamp := time.Now().UTC().UnixNano()
err = db.accessIndex.PutInBatch(batch, internal.IndexItem{
Hash: ref,
AccessTimestamp: accessTimestamp,
})
if err != nil {
return nil, err
}
err = db.gcIndex.PutInBatch(batch, internal.IndexItem{
Hash: item.Hash,
AccessTimestamp: accessTimestamp,
StoreTimestamp: item.StoreTimestamp,
})
if err != nil {
return nil, err
}
err = db.db.WriteBatch(batch)
if err != nil {
return nil, err
}
return storage.NewChunk(item.Hash, item.Data), nil
}
func (db *DB) CollectGarbage() (err error) {
const maxTrashSize = 100
maxRounds := 10 // adbitrary number, needs to be calculated
for roundCount := 0; roundCount < maxRounds; roundCount++ {
var garbageCount int
trash := new(leveldb.Batch)
err = db.gcIndex.IterateAll(func(item internal.IndexItem) (stop bool, err error) {
err = db.retrievalIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
err = db.accessIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
err = db.gcIndex.DeleteInBatch(trash, item)
if err != nil {
return false, err
}
garbageCount++
if garbageCount >= maxTrashSize {
return true, nil
}
return false, nil
})
if err != nil {
return err
}
if garbageCount == 0 {
return nil
}
err = db.db.WriteBatch(trash)
if err != nil {
return err
}
}
return nil
}
func (db *DB) GetSchema() (name string, err error) {
name, err = db.schemaName.Get()
if err == leveldb.ErrNotFound {
return "", nil
}
return name, err
}
func (db *DB) PutSchema(name string) (err error) {
return db.schemaName.Put(name)
}
func (db *DB) Close() {
db.db.Close()
}