diff --git a/swarm/shed/internal/db.go b/swarm/shed/db.go similarity index 62% rename from swarm/shed/internal/db.go rename to swarm/shed/db.go index 02c870e625..987c89dcff 100644 --- a/swarm/shed/internal/db.go +++ b/swarm/shed/db.go @@ -14,7 +14,13 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +// Package shed provides a simple abstraction components to compose +// more complex operations on storage data organized in fields and indexes. +// +// Only type which holds logical information about swarm storage chunks data +// and metadata is IndexItem. This part is not generalized mostly for +// performance reasons. +package shed import ( "github.com/ethereum/go-ethereum/metrics" @@ -23,21 +29,33 @@ import ( "github.com/syndtr/goleveldb/leveldb/opt" ) +// The limit for LevelDB OpenFilesCacheCapacity. const openFileLimit = 128 +// DB provides abstractions over LevelDB in order to +// implement complex structures using fields and ordered indexes. +// It provides a schema functionality to store fields and indexes +// information about naming and types. type DB struct { ldb *leveldb.DB } +// NewDB constructs a new DB and validates the schema +// if it exists in database on the given path. func NewDB(path string) (db *DB, err error) { - ldb, err := leveldb.OpenFile(path, &opt.Options{OpenFilesCacheCapacity: openFileLimit}) + ldb, err := leveldb.OpenFile(path, &opt.Options{ + OpenFilesCacheCapacity: openFileLimit, + }) if err != nil { return nil, err } - db = &DB{ldb: ldb} + db = &DB{ + ldb: ldb, + } if _, err = db.getSchema(); err != nil { if err == leveldb.ErrNotFound { + // save schema with initialized default fields if err = db.putSchema(schema{ Fields: make(map[string]fieldSpec), Indexes: make(map[byte]indexSpec), @@ -51,34 +69,42 @@ func NewDB(path string) (db *DB, err error) { return db, nil } +// Put wraps LevelDB Put method to increment metrics counter. func (db *DB) Put(key []byte, value []byte) (err error) { metrics.GetOrRegisterCounter("DB.put", nil).Inc(1) return db.ldb.Put(key, value, nil) } +// Get wraps LevelDB Get method to increment metrics counter. func (db *DB) Get(key []byte) (value []byte, err error) { metrics.GetOrRegisterCounter("DB.get", nil).Inc(1) return db.ldb.Get(key, nil) } +// Delete wraps LevelDB Delete method to increment metrics counter. func (db *DB) Delete(key []byte) error { + metrics.GetOrRegisterCounter("DB.delete", nil).Inc(1) + return db.ldb.Delete(key, nil) } +// NewIterator wraps LevelDB NewIterator method to increment metrics counter. func (db *DB) NewIterator() iterator.Iterator { metrics.GetOrRegisterCounter("DB.newiterator", nil).Inc(1) return db.ldb.NewIterator(nil, nil) } +// WriteBatch wraps LevelDB Write method to increment metrics counter. func (db *DB) WriteBatch(batch *leveldb.Batch) error { metrics.GetOrRegisterCounter("DB.write", nil).Inc(1) return db.ldb.Write(batch, nil) } +// Close closes LevelDB database. func (db *DB) Close() (err error) { return db.ldb.Close() } diff --git a/swarm/shed/internal/db_test.go b/swarm/shed/db_test.go similarity index 85% rename from swarm/shed/internal/db_test.go rename to swarm/shed/db_test.go index ecb260bf71..45325beeb8 100644 --- a/swarm/shed/internal/db_test.go +++ b/swarm/shed/db_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "io/ioutil" @@ -22,6 +22,8 @@ import ( "testing" ) +// TestNewDB constructs a new DB +// and validates if the schema is initialized properly. func TestNewDB(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -44,6 +46,8 @@ func TestNewDB(t *testing.T) { } } +// TestDB_persistence creates one DB, saves a field and closes that DB. +// Then, it constructs another DB and trues to retrieve the saved value. func TestDB_persistence(t *testing.T) { dir, err := ioutil.TempDir("", "shed-test-persistence") if err != nil { @@ -86,6 +90,9 @@ func TestDB_persistence(t *testing.T) { } } +// newTestDB is a helper function that constructs a +// temporary database and returns a cleanup function that must +// be called to remove the data. func newTestDB(t *testing.T) (db *DB, cleanupFunc func()) { t.Helper() diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go new file mode 100644 index 0000000000..733a972c65 --- /dev/null +++ b/swarm/shed/example_store_test.go @@ -0,0 +1,325 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package shed_test + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "io/ioutil" + "log" + "os" + "time" + + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/syndtr/goleveldb/leveldb" +) + +// Store holds fields and indexes (including their encoding functions) +// and defines operations on them by composing data from them. +// It implements storage.ChunkStore interface. +// It is just an example without any support for parallel operations +// or real world implementation. +type Store struct { + db *shed.DB + + // fields and indexes + schemaName shed.StringField + sizeCounter shed.Uint64Field + accessCounter shed.Uint64Field + retrievalIndex shed.Index + accessIndex shed.Index + gcIndex shed.Index +} + +// New returns new Store. All fields and indexes are initialized +// and possible conflicts with schema from existing database is checked +// automatically. +func New(path string) (s *Store, err error) { + db, err := shed.NewDB(path) + if err != nil { + return nil, err + } + s = &Store{ + db: db, + } + // Identify current storage schema by arbitrary name. + s.schemaName, err = db.NewStringField("schema-name") + if err != nil { + return nil, err + } + // Global ever incrementing index of chunk accesses. + s.accessCounter, err = db.NewUint64Field("access-counter") + if err != nil { + return nil, err + } + // Index storing actual chunk address, data and store timestamp. + s.retrievalIndex, err = db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{ + EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + return fields.Address, nil + }, + DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + e.Address = key + return e, nil + }, + EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) + value = append(b, fields.Data...) + return value, nil + }, + DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) + e.Data = value[8:] + return e, nil + }, + }) + // Index storing access timestamp for a particular address. + // It is needed in order to update gc index keys for iteration order. + s.accessIndex, err = db.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{ + EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + return fields.Address, nil + }, + DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + e.Address = key + return e, nil + }, + EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) + return b, nil + }, + DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) + return e, nil + }, + }) + // Index with keys ordered by access timestamp for garbage collection prioritization. + s.gcIndex, err = db.NewIndex("AccessTimestamp|StoredTimestamp|Address->nil", shed.IndexFuncs{ + EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + b := make([]byte, 16, 16+len(fields.Address)) + binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) + binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) + key = append(b, fields.Address...) + return key, nil + }, + DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) + e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) + e.Address = key[16:] + return e, nil + }, + EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + return e, nil + }, + }) + if err != nil { + return nil, err + } + return s, nil +} + +// Put stores the chunk and sets it store timestamp. +func (s *Store) Put(_ context.Context, ch storage.Chunk) (err error) { + return s.retrievalIndex.Put(shed.IndexItem{ + Address: ch.Address(), + Data: ch.Data(), + StoreTimestamp: time.Now().UTC().UnixNano(), + }) +} + +// Get retrieves a chunk with the provided address. +// It updates access and gc indexes by removing the previous +// items from them and adding new items as keys of index entries +// are changed. +func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, err error) { + batch := new(leveldb.Batch) + + // Get the chunk data and storage timestamp. + item, err := s.retrievalIndex.Get(shed.IndexItem{ + Address: addr, + }) + if err != nil { + if err == leveldb.ErrNotFound { + return nil, storage.ErrChunkNotFound + } + return nil, err + } + + // Get the chunk access timestamp. + accessItem, err := s.accessIndex.Get(shed.IndexItem{ + Address: addr, + }) + switch err { + case nil: + // Remove gc index entry if access timestamp is found. + err = s.gcIndex.DeleteInBatch(batch, shed.IndexItem{ + Address: item.Address, + StoreTimestamp: accessItem.AccessTimestamp, + AccessTimestamp: item.StoreTimestamp, + }) + if err != nil { + return nil, err + } + case leveldb.ErrNotFound: + // Access timestamp is not found. Do not do anything. + // This is the firs get request. + default: + return nil, err + } + + // Specify new access timestamp + accessTimestamp := time.Now().UTC().UnixNano() + + // Put new access timestamp in access index. + err = s.accessIndex.PutInBatch(batch, shed.IndexItem{ + Address: addr, + AccessTimestamp: accessTimestamp, + }) + if err != nil { + return nil, err + } + + // Put new access timestamp in gc index. + err = s.gcIndex.PutInBatch(batch, shed.IndexItem{ + Address: item.Address, + AccessTimestamp: accessTimestamp, + StoreTimestamp: item.StoreTimestamp, + }) + if err != nil { + return nil, err + } + + // Increment access counter. + // Currently this information is not used anywhere. + _, err = s.accessCounter.IncInBatch(batch) + if err != nil { + return nil, err + } + + // Write the batch. + err = s.db.WriteBatch(batch) + if err != nil { + return nil, err + } + + // Return the chunk. + return storage.NewChunk(item.Address, item.Data), nil +} + +// CollectGarbage is an example of index iteration. +// It provides no reliable garbage collection functionality. +func (s *Store) CollectGarbage() (err error) { + const maxTrashSize = 100 + maxRounds := 10 // arbitrary number, needs to be calculated + + // Run a few gc rounds. + for roundCount := 0; roundCount < maxRounds; roundCount++ { + var garbageCount int + // New batch for a new cg round. + trash := new(leveldb.Batch) + // Iterate through all index items and break when needed. + err = s.gcIndex.IterateAll(func(item shed.IndexItem) (stop bool, err error) { + // Remove the chunk. + err = s.retrievalIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + // Remove the element in gc index. + err = s.gcIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + // Remove the relation in access index. + err = s.accessIndex.DeleteInBatch(trash, item) + if err != nil { + return false, err + } + garbageCount++ + if garbageCount >= maxTrashSize { + return true, nil + } + return false, nil + }) + if err != nil { + return err + } + if garbageCount == 0 { + return nil + } + err = s.db.WriteBatch(trash) + if err != nil { + return err + } + } + return nil +} + +// GetSchema is an example of retrieveing the most simple +// string from a database field. +func (s *Store) GetSchema() (name string, err error) { + name, err = s.schemaName.Get() + if err == leveldb.ErrNotFound { + return "", nil + } + return name, err +} + +// GetSchema is an example of storing the most simple +// string in a database field. +func (s *Store) PutSchema(name string) (err error) { + return s.schemaName.Put(name) +} + +// Close closes the underlying database. +func (s *Store) Close() { + s.db.Close() +} + +// Example_store constructs a simple storage implementation using shed package. +func Example_store() { + dir, err := ioutil.TempDir("", "ephemeral") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(dir) + + s, err := New(dir) + if err != nil { + log.Fatal(err) + } + + ch := storage.GenerateRandomChunk(1024) + err = s.Put(context.Background(), ch) + if err != nil { + log.Fatal(err) + } + + got, err := s.Get(context.Background(), ch.Address()) + if err != nil { + log.Fatal(err) + } + + fmt.Println(bytes.Equal(got.Data(), ch.Data())) + + //Output: true +} diff --git a/swarm/shed/internal/field_json.go b/swarm/shed/field_json.go similarity index 77% rename from swarm/shed/internal/field_json.go rename to swarm/shed/field_json.go index bbe634f014..b3274bb1b6 100644 --- a/swarm/shed/internal/field_json.go +++ b/swarm/shed/field_json.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "encoding/json" @@ -22,11 +22,15 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// JSONField is a helper to store complex structure by +// encoding it in JSON format. type JSONField struct { db *DB key []byte } +// NewJSONField returns a new JSONField. +// It validates its name and type against the database schema. func (db *DB) NewJSONField(name string) (f JSONField, err error) { key, err := db.schemaFieldKey(name, "json") if err != nil { @@ -38,18 +42,17 @@ func (db *DB) NewJSONField(name string) (f JSONField, err error) { }, nil } +// Unmarshal unmarshals data fromt he database to a provided val. +// If the data is not found leveldb.ErrNotFound is returned. func (f JSONField) Unmarshal(val interface{}) (err error) { b, err := f.db.Get(f.key) if err != nil { - // Q: should we ignore not found - // if err == leveldb.ErrNotFound { - // return nil - // } return err } return json.Unmarshal(b, val) } +// Put marshals provided val and saves it to the database. func (f JSONField) Put(val interface{}) (err error) { b, err := json.Marshal(val) if err != nil { @@ -58,6 +61,7 @@ func (f JSONField) Put(val interface{}) (err error) { return f.db.Put(f.key, b) } +// PutInBatch marshals provided val and puts it into the batch. func (f JSONField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) { b, err := json.Marshal(val) if err != nil { diff --git a/swarm/shed/internal/field_json_test.go b/swarm/shed/field_json_test.go similarity index 97% rename from swarm/shed/internal/field_json_test.go rename to swarm/shed/field_json_test.go index 3465f7efbf..c9e2cf658d 100644 --- a/swarm/shed/internal/field_json_test.go +++ b/swarm/shed/field_json_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "testing" @@ -22,6 +22,8 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// TestJSONField validates put and unmarshal operations +// of the JSONField. func TestJSONField(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() diff --git a/swarm/shed/internal/field_string.go b/swarm/shed/field_string.go similarity index 75% rename from swarm/shed/internal/field_string.go rename to swarm/shed/field_string.go index 18c8c840d4..c2e5ffa14a 100644 --- a/swarm/shed/internal/field_string.go +++ b/swarm/shed/field_string.go @@ -14,17 +14,21 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "github.com/syndtr/goleveldb/leveldb" ) +// StringField is the most simple field implementation +// that stores an arbitrary string under a specific LevelDB key. type StringField struct { db *DB key []byte } +// NewStringField retruns a new Instance fo StringField. +// It validates its name and type against the database schema. func (db *DB) NewStringField(name string) (f StringField, err error) { key, err := db.schemaFieldKey(name, "string") if err != nil { @@ -36,6 +40,9 @@ func (db *DB) NewStringField(name string) (f StringField, err error) { }, nil } +// Get returns a string value from database. +// If the value is not found, an empty string is returned +// an no error. func (f StringField) Get() (val string, err error) { b, err := f.db.Get(f.key) if err != nil { @@ -47,10 +54,13 @@ func (f StringField) Get() (val string, err error) { return string(b), nil } +// Put stores a string in the database. func (f StringField) Put(val string) (err error) { return f.db.Put(f.key, []byte(val)) } +// PutInBatch stores a string in a batch that can be +// saved later in database. func (f StringField) PutInBatch(batch *leveldb.Batch, val string) { batch.Put(f.key, []byte(val)) } diff --git a/swarm/shed/internal/field_string_test.go b/swarm/shed/field_string_test.go similarity index 96% rename from swarm/shed/internal/field_string_test.go rename to swarm/shed/field_string_test.go index 58d7bdbf8d..4215075bca 100644 --- a/swarm/shed/internal/field_string_test.go +++ b/swarm/shed/field_string_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "testing" @@ -22,6 +22,8 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// TestStringField validates put and get operations +// of the StringField. func TestStringField(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() diff --git a/swarm/shed/internal/field_uint64.go b/swarm/shed/field_uint64.go similarity index 71% rename from swarm/shed/internal/field_uint64.go rename to swarm/shed/field_uint64.go index 3d92a09ee3..80e0069ae4 100644 --- a/swarm/shed/internal/field_uint64.go +++ b/swarm/shed/field_uint64.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "encoding/binary" @@ -22,11 +22,15 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// Uint64Field provides a way to have a simple counter in the database. +// It transparently encodes uint64 type value to bytes. type Uint64Field struct { db *DB key []byte } +// NewUint64Field returns a new Uint64Field. +// It validates its name and type against the database schema. func (db *DB) NewUint64Field(name string) (f Uint64Field, err error) { key, err := db.schemaFieldKey(name, "uint64") if err != nil { @@ -38,6 +42,9 @@ func (db *DB) NewUint64Field(name string) (f Uint64Field, err error) { }, nil } +// Get retrieves a uint64 value from the database. +// If the value is not found in the database a 0 value +// is returned and no error. func (f Uint64Field) Get() (val uint64, err error) { b, err := f.db.Get(f.key) if err != nil { @@ -49,14 +56,19 @@ func (f Uint64Field) Get() (val uint64, err error) { return binary.BigEndian.Uint64(b), nil } +// Put encodes uin64 value and stores it in the database. func (f Uint64Field) Put(val uint64) (err error) { return f.db.Put(f.key, encodeUint64(val)) } +// PutInBatch stores a uint64 value in a batch +// that can be saved later in the database. func (f Uint64Field) PutInBatch(batch *leveldb.Batch, val uint64) { batch.Put(f.key, encodeUint64(val)) } +// Inc increments a uint64 value in the database. +// This operation is not goroutine save. func (f Uint64Field) Inc() (val uint64, err error) { val, err = f.Get() if err != nil { @@ -70,6 +82,9 @@ func (f Uint64Field) Inc() (val uint64, err error) { return val, f.Put(val) } +// IncInBatch increments a uint64 value in the batch +// by retreiving a value from the database, not the same batch. +// This operation is not goroutine save. func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) { val, err = f.Get() if err != nil { @@ -84,6 +99,8 @@ func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) { return val, nil } +// encode transforms uint64 to 8 byte long +// slice in big endian encoding. func encodeUint64(val uint64) (b []byte) { b = make([]byte, 8) binary.BigEndian.PutUint64(b, val) diff --git a/swarm/shed/internal/field_uint64_test.go b/swarm/shed/field_uint64_test.go similarity index 93% rename from swarm/shed/internal/field_uint64_test.go rename to swarm/shed/field_uint64_test.go index 436971d251..69ade71ba3 100644 --- a/swarm/shed/internal/field_uint64_test.go +++ b/swarm/shed/field_uint64_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "testing" @@ -22,6 +22,8 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// TestUint64Field validates put and get operations +// of the Uint64Field. func TestUint64Field(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -36,7 +38,7 @@ func TestUint64Field(t *testing.T) { if err != nil { t.Fatal(err) } - var want uint64 = 0 + var want uint64 if got != want { t.Errorf("got uint64 %v, want %v", got, want) } @@ -107,6 +109,8 @@ func TestUint64Field(t *testing.T) { }) } +// TestUint64Field_Inc validates Inc operation +// of the Uint64Field. func TestUint64Field_Inc(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -135,6 +139,8 @@ func TestUint64Field_Inc(t *testing.T) { } } +// TestUint64Field_IncInBatch validates IncInBatch operation +// of the Uint64Field. func TestUint64Field_IncInBatch(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() diff --git a/swarm/shed/internal/index.go b/swarm/shed/index.go similarity index 62% rename from swarm/shed/internal/index.go rename to swarm/shed/index.go index 3d86c83aa5..d1cf7a757f 100644 --- a/swarm/shed/internal/index.go +++ b/swarm/shed/index.go @@ -14,22 +14,37 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "github.com/syndtr/goleveldb/leveldb" ) +// IndexItem holds fields relevant to Swarm Chunk data and metadata. +// All information required for swarm storage and operations +// on that storage must be defined here. +// This structure is logically connected to swarm storage, +// the only part of this package that is not generalized, +// mostly for performance reasons. +// +// IndexItem is a type that is used for retrieving, storing and encoding +// chunk data and metadata. It is passed as an argument to Index encoding +// functions, get function and put function. +// But it is also returned with additional data from get function call +// and as the argument in iterator function definition. type IndexItem struct { - Hash []byte + Address []byte Data []byte AccessTimestamp int64 StoreTimestamp int64 } +// Join is a helper method to construct a new +// IndexItem by filling up fields with default values +// of a particular IndexItem with values from another one. func (i IndexItem) Join(i2 IndexItem) (new IndexItem) { - if i.Hash == nil { - i.Hash = i2.Hash + if i.Address == nil { + i.Address = i2.Address } if i.Data == nil { i.Data = i2.Data @@ -43,6 +58,12 @@ func (i IndexItem) Join(i2 IndexItem) (new IndexItem) { return i } +// Index represents a set of LevelDB key value pairs that have common +// prefix. It holds functions for encoding and decoding keys and values +// to provide transparent actions on saved data which inclide: +// - getting a particular IndexItem +// - saving a particular IndexItem +// - iterating over a sorted LevelDB keys type Index struct { db *DB prefix []byte @@ -52,6 +73,8 @@ type Index struct { decodeValueFunc func(value []byte) (e IndexItem, err error) } +// IndexFuncs structure defines functions for encoding and decoding +// LevelDB keys and values for a specific index. type IndexFuncs struct { EncodeKey func(fields IndexItem) (key []byte, err error) DecodeKey func(key []byte) (e IndexItem, err error) @@ -59,8 +82,11 @@ type IndexFuncs struct { DecodeValue func(value []byte) (e IndexItem, err error) } +// NewIndex returns a new Index instance with defined name and +// encoding functions. The name must be unique and will be validated +// on database schema for a key prefix byte. func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { - id, err := db.schemaIndexID(name) + id, err := db.schemaIndexPrefix(name) if err != nil { return f, err } @@ -83,6 +109,9 @@ func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { }, nil } +// Get accepts key fields represented as IndexItem to retrieve a +// value from the index and return maximum available information +// from the index represented as another IndexItem. func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { @@ -99,6 +128,8 @@ func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) { return out.Join(keyFields), nil } +// Put accepts IndexItem to encode information from it +// and save it to the database. func (f Index) Put(i IndexItem) (err error) { key, err := f.encodeKeyFunc(i) if err != nil { @@ -111,6 +142,9 @@ func (f Index) Put(i IndexItem) (err error) { return f.db.Put(key, value) } +// PutInBatch is the same as Put method, but it just +// saves the key/value pair to the batch instead +// directly to the database. func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) { key, err := f.encodeKeyFunc(i) if err != nil { @@ -124,6 +158,8 @@ func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) { return nil } +// Delete accepts IndexItem to remove a key/value pair +// form the database based on its fields. func (f Index) Delete(keyFields IndexItem) (err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { @@ -132,6 +168,8 @@ func (f Index) Delete(keyFields IndexItem) (err error) { return f.db.Delete(key) } +// DeleteInBatch is the same as Delete just the operation +// is performed on the batch instead on the database. func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { @@ -141,9 +179,15 @@ func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err err return nil } -type IterFunc func(item IndexItem) (stop bool, err error) +// IndexIterFunc is a callback on every IndexItem that is decoded +// by iterating on an Index keys. +// By returning a true for stop variable, iteration will +// stop, and by returning the error, that error will be +// propagated to the called iterator method on Index. +type IndexIterFunc func(item IndexItem) (stop bool, err error) -func (f Index) IterateAll(fn IterFunc) (err error) { +// IterateAll iterates over all keys of the Index. +func (f Index) IterateAll(fn IndexIterFunc) (err error) { it := f.db.NewIterator() defer it.Release() @@ -171,7 +215,9 @@ func (f Index) IterateAll(fn IterFunc) (err error) { return it.Error() } -func (f Index) IterateFrom(start IndexItem, fn IterFunc) (err error) { +// IterateFrom iterates over Index keys starting from the key +// encoded from the provided IndexItem. +func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) { startKey, err := f.encodeKeyFunc(start) if err != nil { return err diff --git a/swarm/shed/internal/index_test.go b/swarm/shed/index_test.go similarity index 82% rename from swarm/shed/internal/index_test.go rename to swarm/shed/index_test.go index ebff569381..4e3eba326a 100644 --- a/swarm/shed/internal/index_test.go +++ b/swarm/shed/index_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "bytes" @@ -27,12 +27,13 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +// Index functions for the index that is used in tests in this file. var retrievalIndexFuncs = IndexFuncs{ EncodeKey: func(fields IndexItem) (key []byte, err error) { - return fields.Hash, nil + return fields.Address, nil }, DecodeKey: func(key []byte) (e IndexItem, err error) { - e.Hash = key + e.Address = key return e, nil }, EncodeValue: func(fields IndexItem) (value []byte, err error) { @@ -48,6 +49,7 @@ var retrievalIndexFuncs = IndexFuncs{ }, } +// TestIndex validates put, get and delete functions of the index. func TestIndex(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -59,7 +61,7 @@ func TestIndex(t *testing.T) { t.Run("put", func(t *testing.T) { want := IndexItem{ - Hash: []byte("put-hash"), + Address: []byte("put-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -69,7 +71,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -78,7 +80,7 @@ func TestIndex(t *testing.T) { t.Run("overwrite", func(t *testing.T) { want := IndexItem{ - Hash: []byte("put-hash"), + Address: []byte("put-hash"), Data: []byte("New DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -88,7 +90,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -99,7 +101,7 @@ func TestIndex(t *testing.T) { t.Run("put in batch", func(t *testing.T) { want := IndexItem{ - Hash: []byte("put-in-batch-hash"), + Address: []byte("put-in-batch-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -111,7 +113,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -120,7 +122,7 @@ func TestIndex(t *testing.T) { t.Run("overwrite", func(t *testing.T) { want := IndexItem{ - Hash: []byte("put-in-batch-hash"), + Address: []byte("put-in-batch-hash"), Data: []byte("New DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -132,7 +134,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -143,7 +145,7 @@ func TestIndex(t *testing.T) { t.Run("delete", func(t *testing.T) { want := IndexItem{ - Hash: []byte("delete-hash"), + Address: []byte("delete-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -153,7 +155,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -161,14 +163,14 @@ func TestIndex(t *testing.T) { checkIndexItem(t, got, want) err = index.Delete(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) } got, err = index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != leveldb.ErrNotFound { t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) @@ -177,7 +179,7 @@ func TestIndex(t *testing.T) { t.Run("delete in batch", func(t *testing.T) { want := IndexItem{ - Hash: []byte("delete-in-batch-hash"), + Address: []byte("delete-in-batch-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), } @@ -187,7 +189,7 @@ func TestIndex(t *testing.T) { t.Fatal(err) } got, err := index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != nil { t.Fatal(err) @@ -196,7 +198,7 @@ func TestIndex(t *testing.T) { batch := new(leveldb.Batch) index.DeleteInBatch(batch, IndexItem{ - Hash: want.Hash, + Address: want.Address, }) err = db.WriteBatch(batch) if err != nil { @@ -204,7 +206,7 @@ func TestIndex(t *testing.T) { } got, err = index.Get(IndexItem{ - Hash: want.Hash, + Address: want.Address, }) if err != leveldb.ErrNotFound { t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound) @@ -212,6 +214,7 @@ func TestIndex(t *testing.T) { }) } +// TestIndex_iterate validates index iterator functions for correctness. func TestIndex_iterate(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -223,24 +226,24 @@ func TestIndex_iterate(t *testing.T) { items := []IndexItem{ { - Hash: []byte("iterate-hash-01"), - Data: []byte("data80"), + Address: []byte("iterate-hash-01"), + Data: []byte("data80"), }, { - Hash: []byte("iterate-hash-03"), - Data: []byte("data22"), + Address: []byte("iterate-hash-03"), + Data: []byte("data22"), }, { - Hash: []byte("iterate-hash-05"), - Data: []byte("data41"), + Address: []byte("iterate-hash-05"), + Data: []byte("data41"), }, { - Hash: []byte("iterate-hash-02"), - Data: []byte("data84"), + Address: []byte("iterate-hash-02"), + Data: []byte("data84"), }, { - Hash: []byte("iterate-hash-06"), - Data: []byte("data1"), + Address: []byte("iterate-hash-06"), + Data: []byte("data1"), }, } batch := new(leveldb.Batch) @@ -252,8 +255,8 @@ func TestIndex_iterate(t *testing.T) { t.Fatal(err) } item04 := IndexItem{ - Hash: []byte("iterate-hash-04"), - Data: []byte("data0"), + Address: []byte("iterate-hash-04"), + Data: []byte("data0"), } err = index.Put(item04) if err != nil { @@ -262,7 +265,7 @@ func TestIndex_iterate(t *testing.T) { items = append(items, item04) sort.SliceStable(items, func(i, j int) bool { - return bytes.Compare(items[i].Hash, items[j].Hash) < 0 + return bytes.Compare(items[i].Address, items[j].Address) < 0 }) t.Run("all", func(t *testing.T) { @@ -331,8 +334,8 @@ func TestIndex_iterate(t *testing.T) { } secondIndexItem := IndexItem{ - Hash: []byte("iterate-hash-10"), - Data: []byte("data-second"), + Address: []byte("iterate-hash-10"), + Data: []byte("data-second"), } err = secondIndex.Put(secondIndexItem) if err != nil { @@ -368,11 +371,12 @@ func TestIndex_iterate(t *testing.T) { }) } +// checkIndexItem is a test helper function that compares if two Index items are the same. func checkIndexItem(t *testing.T, got, want IndexItem) { t.Helper() - if !bytes.Equal(got.Hash, want.Hash) { - t.Errorf("got hash %q, expected %q", string(got.Hash), string(want.Hash)) + if !bytes.Equal(got.Address, want.Address) { + t.Errorf("got hash %q, expected %q", string(got.Address), string(want.Address)) } if !bytes.Equal(got.Data, want.Data) { t.Errorf("got data %q, expected %q", string(got.Data), string(want.Data)) diff --git a/swarm/shed/internal/example_dbstore_test.go b/swarm/shed/internal/example_dbstore_test.go deleted file mode 100644 index a9fd037271..0000000000 --- a/swarm/shed/internal/example_dbstore_test.go +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package internal_test - -import ( - "bytes" - "context" - "encoding/binary" - "fmt" - "io/ioutil" - "log" - "os" - "time" - - "github.com/ethereum/go-ethereum/swarm/shed/internal" - "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/syndtr/goleveldb/leveldb" -) - -// DB is just an example for composing indexes. -type DB struct { - db *internal.DB - - // fields and indexes - schemaName internal.StringField - sizeCounter internal.Uint64Field - accessCounter internal.Uint64Field - retrievalIndex internal.Index - accessIndex internal.Index - gcIndex internal.Index -} - -func New(path string) (db *DB, err error) { - idb, err := internal.NewDB(path) - if err != nil { - return nil, err - } - db = &DB{ - db: idb, - } - db.schemaName, err = idb.NewStringField("schema-name") - if err != nil { - return nil, err - } - db.sizeCounter, err = idb.NewUint64Field("size-counter") - if err != nil { - return nil, err - } - db.accessCounter, err = idb.NewUint64Field("access-counter") - if err != nil { - return nil, err - } - db.retrievalIndex, err = idb.NewIndex("Hash->StoreTimestamp|Data", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - return fields.Hash, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.Hash = key - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) - value = append(b, fields.Data...) - return value, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) - e.Data = value[8:] - return e, nil - }, - }) - db.accessIndex, err = idb.NewIndex("Hash->AccessTimestamp", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - return fields.Hash, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.Hash = key - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) - return b, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) - return e, nil - }, - }) - db.gcIndex, err = idb.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - b := make([]byte, 16, 16+len(fields.Hash)) - binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) - binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) - key = append(b, fields.Hash...) - return key, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) - e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) - e.Hash = key[16:] - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - return nil, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - return e, nil - }, - }) - if err != nil { - return nil, err - } - return db, nil -} - -func (db *DB) Put(_ context.Context, ch storage.Chunk) (err error) { - return db.retrievalIndex.Put(internal.IndexItem{ - Hash: ch.Address(), - Data: ch.Data(), - StoreTimestamp: time.Now().UTC().UnixNano(), - }) -} - -func (db *DB) Get(_ context.Context, ref storage.Address) (c storage.Chunk, err error) { - batch := new(leveldb.Batch) - - item, err := db.retrievalIndex.Get(internal.IndexItem{ - Hash: ref, - }) - if err != nil { - if err == leveldb.ErrNotFound { - return nil, storage.ErrChunkNotFound - } - return nil, err - } - - accessItem, err := db.accessIndex.Get(internal.IndexItem{ - Hash: ref, - }) - switch err { - case nil: - err = db.gcIndex.DeleteInBatch(batch, internal.IndexItem{ - Hash: item.Hash, - StoreTimestamp: accessItem.AccessTimestamp, - AccessTimestamp: item.StoreTimestamp, - }) - if err != nil { - return nil, err - } - case leveldb.ErrNotFound: - default: - return nil, err - } - - accessTimestamp := time.Now().UTC().UnixNano() - - err = db.accessIndex.PutInBatch(batch, internal.IndexItem{ - Hash: ref, - AccessTimestamp: accessTimestamp, - }) - if err != nil { - return nil, err - } - - err = db.gcIndex.PutInBatch(batch, internal.IndexItem{ - Hash: item.Hash, - AccessTimestamp: accessTimestamp, - StoreTimestamp: item.StoreTimestamp, - }) - if err != nil { - return nil, err - } - - err = db.db.WriteBatch(batch) - if err != nil { - return nil, err - } - - return storage.NewChunk(item.Hash, item.Data), nil -} - -func (db *DB) CollectGarbage() (err error) { - const maxTrashSize = 100 - maxRounds := 10 // adbitrary number, needs to be calculated - - for roundCount := 0; roundCount < maxRounds; roundCount++ { - var garbageCount int - trash := new(leveldb.Batch) - err = db.gcIndex.IterateAll(func(item internal.IndexItem) (stop bool, err error) { - err = db.retrievalIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - err = db.accessIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - err = db.gcIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - garbageCount++ - if garbageCount >= maxTrashSize { - return true, nil - } - return false, nil - }) - if err != nil { - return err - } - if garbageCount == 0 { - return nil - } - err = db.db.WriteBatch(trash) - if err != nil { - return err - } - } - return nil -} - -func (db *DB) GetSchema() (name string, err error) { - name, err = db.schemaName.Get() - if err == leveldb.ErrNotFound { - return "", nil - } - return name, err -} - -func (db *DB) PutSchema(name string) (err error) { - return db.schemaName.Put(name) -} - -func (db *DB) Close() { - db.db.Close() -} - -func Example_dbstore() { - dir, err := ioutil.TempDir("", "ephemeral") - if err != nil { - log.Fatal(err) - } - defer os.RemoveAll(dir) - - db, err := New(dir) - if err != nil { - log.Fatal(err) - } - - ch := storage.GenerateRandomChunk(1024) - err = db.Put(context.Background(), ch) - if err != nil { - log.Fatal(err) - } - - got, err := db.Get(context.Background(), ch.Address()) - if err != nil { - log.Fatal(err) - } - - fmt.Println(bytes.Equal(got.Data(), ch.Data())) - - //Output: true -} diff --git a/swarm/shed/internal/schema.go b/swarm/shed/schema.go similarity index 63% rename from swarm/shed/internal/schema.go rename to swarm/shed/schema.go index 9440ba8955..579d058aba 100644 --- a/swarm/shed/internal/schema.go +++ b/swarm/shed/schema.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "encoding/json" @@ -23,24 +23,37 @@ import ( ) var ( - keySchema = []byte{0} - keyPrefixFields byte = 1 - keyPrefixIndexStart byte = 2 // Q: or maybe 7, to have more space for potential specific perfixes + // LevelDB key value for storing the schema. + keySchema = []byte{0} + // LevelDB key prefix for all field type. + // LevelDB keys will be constructed by appending name values to this prefix. + keyPrefixFields byte = 1 + // LevelDB key prefix from which indexing keys start. + // Every index has its own key prefix and this value defines the first one. + keyPrefixIndexStart byte = 2 // Q: or maybe a higher number like 7, to have more space for potential specific perfixes ) +// schema is used to serialize known database structure information. type schema struct { - Fields map[string]fieldSpec `json:"fields"` - Indexes map[byte]indexSpec `json:"indexes"` + Fields map[string]fieldSpec `json:"fields"` // keys are field names + Indexes map[byte]indexSpec `json:"indexes"` // keys are index prefix bytes } +// fieldSpec holds information about a particular field. +// It does not need Name field as it is contained in the +// schema.Field map key. type fieldSpec struct { Type string `json:"type"` } +// indxSpec holds information about a particular index. +// It does not contain index type, as indexes do not have type. type indexSpec struct { Name string `json:"name"` } +// schemaFieldKey retrives the complete LevelDB key for +// a particular field form the schema definition. func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) { if name == "" { return nil, errors.New("filed name can not be blank") @@ -73,7 +86,9 @@ func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) { return append([]byte{keyPrefixFields}, []byte(name)...), nil } -func (db *DB) schemaIndexID(name string) (id byte, err error) { +// schemaIndexID retrieves the complete LevelDB prefix for +// a particular index. +func (db *DB) schemaIndexPrefix(name string) (id byte, err error) { if name == "" { return 0, errors.New("index name can not be blank") } @@ -97,6 +112,8 @@ func (db *DB) schemaIndexID(name string) (id byte, err error) { return id, db.putSchema(s) } +// getSchema retrieves the complete schema from +// the database. func (db *DB) getSchema() (s schema, err error) { b, err := db.Get(keySchema) if err != nil { @@ -106,6 +123,8 @@ func (db *DB) getSchema() (s schema, err error) { return s, err } +// putSchema stores the complete schema to +// the database. func (db *DB) putSchema(s schema) (err error) { b, err := json.Marshal(s) if err != nil { diff --git a/swarm/shed/internal/schema_test.go b/swarm/shed/schema_test.go similarity index 87% rename from swarm/shed/internal/schema_test.go rename to swarm/shed/schema_test.go index 8b3682a8e9..a0c1838c8b 100644 --- a/swarm/shed/internal/schema_test.go +++ b/swarm/shed/schema_test.go @@ -14,14 +14,15 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package internal +package shed import ( "bytes" "testing" ) -func TestSchema_schemaFieldKey(t *testing.T) { +// TestDB_schemaFieldKey validates correctness of schemaFieldKey. +func TestDB_schemaFieldKey(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -86,17 +87,18 @@ func TestSchema_schemaFieldKey(t *testing.T) { }) } -func TestSchema_schemaIndexID(t *testing.T) { +// TestDB_schemaIndexPrefix validates correctness of schemaIndexPrefix. +func TestDB_schemaIndexPrefix(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() t.Run("same name", func(t *testing.T) { - id1, err := db.schemaIndexID("test") + id1, err := db.schemaIndexPrefix("test") if err != nil { t.Fatal(err) } - id2, err := db.schemaIndexID("test") + id2, err := db.schemaIndexPrefix("test") if err != nil { t.Fatal(err) } @@ -107,12 +109,12 @@ func TestSchema_schemaIndexID(t *testing.T) { }) t.Run("different names", func(t *testing.T) { - id1, err := db.schemaIndexID("test1") + id1, err := db.schemaIndexPrefix("test1") if err != nil { t.Fatal(err) } - id2, err := db.schemaIndexID("test2") + id2, err := db.schemaIndexPrefix("test2") if err != nil { t.Fatal(err) } diff --git a/swarm/shed/shed.go b/swarm/shed/shed.go deleted file mode 100644 index 4f65d4ed81..0000000000 --- a/swarm/shed/shed.go +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package shed - -import ( - "context" - "encoding/binary" - "time" - - "github.com/ethereum/go-ethereum/swarm/shed/internal" - "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/syndtr/goleveldb/leveldb" -) - -// DB is just an example for composing indexes. -type DB struct { - db *internal.DB - - // fields and indexes - schemaName internal.StringField - sizeCounter internal.Uint64Field - accessCounter internal.Uint64Field - retrievalIndex internal.Index - accessIndex internal.Index - gcIndex internal.Index -} - -func New(path string) (db *DB, err error) { - idb, err := internal.NewDB(path) - if err != nil { - return nil, err - } - db = &DB{ - db: idb, - } - db.schemaName, err = idb.NewStringField("schema-name") - if err != nil { - return nil, err - } - db.sizeCounter, err = idb.NewUint64Field("size-counter") - if err != nil { - return nil, err - } - db.accessCounter, err = idb.NewUint64Field("access-counter") - if err != nil { - return nil, err - } - db.retrievalIndex, err = idb.NewIndex("Hash->StoreTimestamp|Data", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - return fields.Hash, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.Hash = key - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) - value = append(b, fields.Data...) - return value, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) - e.Data = value[8:] - return e, nil - }, - }) - db.accessIndex, err = idb.NewIndex("Hash->AccessTimestamp", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - return fields.Hash, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.Hash = key - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - b := make([]byte, 8) - binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) - return b, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) - return e, nil - }, - }) - db.gcIndex, err = idb.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", internal.IndexFuncs{ - EncodeKey: func(fields internal.IndexItem) (key []byte, err error) { - b := make([]byte, 16, 16+len(fields.Hash)) - binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) - binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) - key = append(b, fields.Hash...) - return key, nil - }, - DecodeKey: func(key []byte) (e internal.IndexItem, err error) { - e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) - e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) - e.Hash = key[16:] - return e, nil - }, - EncodeValue: func(fields internal.IndexItem) (value []byte, err error) { - return nil, nil - }, - DecodeValue: func(value []byte) (e internal.IndexItem, err error) { - return e, nil - }, - }) - if err != nil { - return nil, err - } - return db, nil -} - -func (db *DB) Put(_ context.Context, ch storage.Chunk) (err error) { - return db.retrievalIndex.Put(internal.IndexItem{ - Hash: ch.Address(), - Data: ch.Data(), - StoreTimestamp: time.Now().UTC().UnixNano(), - }) -} - -func (db *DB) Get(_ context.Context, ref storage.Address) (c storage.Chunk, err error) { - batch := new(leveldb.Batch) - - item, err := db.retrievalIndex.Get(internal.IndexItem{ - Hash: ref, - }) - if err != nil { - if err == leveldb.ErrNotFound { - return nil, storage.ErrChunkNotFound - } - return nil, err - } - - accessItem, err := db.accessIndex.Get(internal.IndexItem{ - Hash: ref, - }) - switch err { - case nil: - err = db.gcIndex.DeleteInBatch(batch, internal.IndexItem{ - Hash: item.Hash, - StoreTimestamp: accessItem.AccessTimestamp, - AccessTimestamp: item.StoreTimestamp, - }) - if err != nil { - return nil, err - } - case leveldb.ErrNotFound: - default: - return nil, err - } - - accessTimestamp := time.Now().UTC().UnixNano() - - err = db.accessIndex.PutInBatch(batch, internal.IndexItem{ - Hash: ref, - AccessTimestamp: accessTimestamp, - }) - if err != nil { - return nil, err - } - - err = db.gcIndex.PutInBatch(batch, internal.IndexItem{ - Hash: item.Hash, - AccessTimestamp: accessTimestamp, - StoreTimestamp: item.StoreTimestamp, - }) - if err != nil { - return nil, err - } - - err = db.db.WriteBatch(batch) - if err != nil { - return nil, err - } - - return storage.NewChunk(item.Hash, item.Data), nil -} - -func (db *DB) CollectGarbage() (err error) { - const maxTrashSize = 100 - maxRounds := 10 // adbitrary number, needs to be calculated - - for roundCount := 0; roundCount < maxRounds; roundCount++ { - var garbageCount int - trash := new(leveldb.Batch) - err = db.gcIndex.IterateAll(func(item internal.IndexItem) (stop bool, err error) { - err = db.retrievalIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - err = db.accessIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - err = db.gcIndex.DeleteInBatch(trash, item) - if err != nil { - return false, err - } - garbageCount++ - if garbageCount >= maxTrashSize { - return true, nil - } - return false, nil - }) - if err != nil { - return err - } - if garbageCount == 0 { - return nil - } - err = db.db.WriteBatch(trash) - if err != nil { - return err - } - } - return nil -} - -func (db *DB) GetSchema() (name string, err error) { - name, err = db.schemaName.Get() - if err == leveldb.ErrNotFound { - return "", nil - } - return name, err -} - -func (db *DB) PutSchema(name string) (err error) { - return db.schemaName.Put(name) -} - -func (db *DB) Close() { - db.db.Close() -}