swarm/shed: initial implementation

swarm/shed/internal/db.go

@@ -0,0 +1,84 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+const openFileLimit = 128
+
+type DB struct {
+	ldb *leveldb.DB
+}
+
+func NewDB(path string) (db *DB, err error) {
+	ldb, err := leveldb.OpenFile(path, &opt.Options{OpenFilesCacheCapacity: openFileLimit})
+	if err != nil {
+		return nil, err
+	}
+	db = &DB{ldb: ldb}
+
+	if _, err = db.getSchema(); err != nil {
+		if err == leveldb.ErrNotFound {
+			if err = db.putSchema(schema{
+				Fields:  make(map[string]fieldSpec),
+				Indexes: make(map[byte]indexSpec),
+			}); err != nil {
+				return nil, err
+			}
+		} else {
+			return nil, err
+		}
+	}
+	return db, nil
+}
+
+func (db *DB) Put(key []byte, value []byte) (err error) {
+	metrics.GetOrRegisterCounter("DB.put", nil).Inc(1)
+
+	return db.ldb.Put(key, value, nil)
+}
+
+func (db *DB) Get(key []byte) (value []byte, err error) {
+	metrics.GetOrRegisterCounter("DB.get", nil).Inc(1)
+
+	return db.ldb.Get(key, nil)
+}
+
+func (db *DB) Delete(key []byte) error {
+	return db.ldb.Delete(key, nil)
+}
+
+func (db *DB) NewIterator() iterator.Iterator {
+	metrics.GetOrRegisterCounter("DB.newiterator", nil).Inc(1)
+
+	return db.ldb.NewIterator(nil, nil)
+}
+
+func (db *DB) WriteBatch(batch *leveldb.Batch) error {
+	metrics.GetOrRegisterCounter("DB.write", nil).Inc(1)
+
+	return db.ldb.Write(batch, nil)
+}
+
+func (db *DB) Close() (err error) {
+	return db.ldb.Close()
+}

swarm/shed/internal/db_test.go

@@ -0,0 +1,103 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestNewDB(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	s, err := db.getSchema()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if s.Fields == nil {
+		t.Error("schema fields are empty")
+	}
+	if len(s.Fields) != 0 {
+		t.Errorf("got schema fields length %v, want %v", len(s.Fields), 0)
+	}
+	if s.Indexes == nil {
+		t.Error("schema indexes are empty")
+	}
+	if len(s.Indexes) != 0 {
+		t.Errorf("got schema indexes length %v, want %v", len(s.Indexes), 0)
+	}
+}
+
+func TestDB_persistence(t *testing.T) {
+	dir, err := ioutil.TempDir("", "shed-test-persistence")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	db, err := NewDB(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	stringField, err := db.NewStringField("preserve-me")
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "persistent value"
+	err = stringField.Put(want)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = db.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db2, err := NewDB(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	stringField2, err := db2.NewStringField("preserve-me")
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := stringField2.Get()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Errorf("got string %q, want %q", got, want)
+	}
+}
+
+func newTestDB(t *testing.T) (db *DB, cleanupFunc func()) {
+	t.Helper()
+
+	dir, err := ioutil.TempDir("", "shed-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	cleanupFunc = func() { os.RemoveAll(dir) }
+	db, err = NewDB(dir)
+	if err != nil {
+		cleanupFunc()
+		t.Fatal(err)
+	}
+	return db, cleanupFunc
+}

swarm/shed/internal/field_json.go

@@ -0,0 +1,68 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"encoding/json"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type JSONField struct {
+	db  *DB
+	key []byte
+}
+
+func (db *DB) NewJSONField(name string) (f JSONField, err error) {
+	key, err := db.schemaFieldKey(name, "json")
+	if err != nil {
+		return f, err
+	}
+	return JSONField{
+		db:  db,
+		key: key,
+	}, nil
+}
+
+func (f JSONField) Unmarshal(val interface{}) (err error) {
+	b, err := f.db.Get(f.key)
+	if err != nil {
+		// Q: should we ignore not found
+		// if err == leveldb.ErrNotFound {
+		// 	return nil
+		// }
+		return err
+	}
+	return json.Unmarshal(b, val)
+}
+
+func (f JSONField) Put(val interface{}) (err error) {
+	b, err := json.Marshal(val)
+	if err != nil {
+		return err
+	}
+	return f.db.Put(f.key, b)
+}
+
+func (f JSONField) PutInBatch(batch *leveldb.Batch, val interface{}) (err error) {
+	b, err := json.Marshal(val)
+	if err != nil {
+		return err
+	}
+	batch.Put(f.key, b)
+	return nil
+}

swarm/shed/internal/field_json_test.go

@@ -0,0 +1,125 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+func TestJSONField(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	complexField, err := db.NewJSONField("complex-field")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	type complexStructure struct {
+		A string
+	}
+
+	t.Run("unmarshal empty", func(t *testing.T) {
+		var s complexStructure
+		err := complexField.Unmarshal(&s)
+		if err != leveldb.ErrNotFound {
+			t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
+		}
+		want := ""
+		if s.A != want {
+			t.Errorf("got string %q, want %q", s.A, want)
+		}
+	})
+
+	t.Run("put", func(t *testing.T) {
+		want := complexStructure{
+			A: "simple string value",
+		}
+		err = complexField.Put(want)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got complexStructure
+		err = complexField.Unmarshal(&got)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got.A != want.A {
+			t.Errorf("got string %q, want %q", got.A, want.A)
+		}
+
+		t.Run("overwrite", func(t *testing.T) {
+			want := complexStructure{
+				A: "overwritten string value",
+			}
+			err = complexField.Put(want)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var got complexStructure
+			err = complexField.Unmarshal(&got)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got.A != want.A {
+				t.Errorf("got string %q, want %q", got.A, want.A)
+			}
+		})
+	})
+
+	t.Run("put in batch", func(t *testing.T) {
+		batch := new(leveldb.Batch)
+		want := complexStructure{
+			A: "simple string batch value",
+		}
+		complexField.PutInBatch(batch, want)
+		err = db.WriteBatch(batch)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var got complexStructure
+		err := complexField.Unmarshal(&got)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got.A != want.A {
+			t.Errorf("got string %q, want %q", got, want)
+		}
+
+		t.Run("overwrite", func(t *testing.T) {
+			batch := new(leveldb.Batch)
+			want := complexStructure{
+				A: "overwritten string batch value",
+			}
+			complexField.PutInBatch(batch, want)
+			err = db.WriteBatch(batch)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var got complexStructure
+			err := complexField.Unmarshal(&got)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got.A != want.A {
+				t.Errorf("got string %q, want %q", got, want)
+			}
+		})
+	})
+}

swarm/shed/internal/field_string.go

@@ -0,0 +1,56 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type StringField struct {
+	db  *DB
+	key []byte
+}
+
+func (db *DB) NewStringField(name string) (f StringField, err error) {
+	key, err := db.schemaFieldKey(name, "string")
+	if err != nil {
+		return f, err
+	}
+	return StringField{
+		db:  db,
+		key: key,
+	}, nil
+}
+
+func (f StringField) Get() (val string, err error) {
+	b, err := f.db.Get(f.key)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return "", nil
+		}
+		return "", err
+	}
+	return string(b), nil
+}
+
+func (f StringField) Put(val string) (err error) {
+	return f.db.Put(f.key, []byte(val))
+}
+
+func (f StringField) PutInBatch(batch *leveldb.Batch, val string) {
+	batch.Put(f.key, []byte(val))
+}

swarm/shed/internal/field_string_test.go

@@ -0,0 +1,108 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+func TestStringField(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	simpleString, err := db.NewStringField("simple-string")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("get empty", func(t *testing.T) {
+		got, err := simpleString.Get()
+		if err != nil {
+			t.Fatal(err)
+		}
+		want := ""
+		if got != want {
+			t.Errorf("got string %q, want %q", got, want)
+		}
+	})
+
+	t.Run("put", func(t *testing.T) {
+		want := "simple string value"
+		err = simpleString.Put(want)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := simpleString.Get()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got != want {
+			t.Errorf("got string %q, want %q", got, want)
+		}
+
+		t.Run("overwrite", func(t *testing.T) {
+			want := "overwritten string value"
+			err = simpleString.Put(want)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := simpleString.Get()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != want {
+				t.Errorf("got string %q, want %q", got, want)
+			}
+		})
+	})
+
+	t.Run("put in batch", func(t *testing.T) {
+		batch := new(leveldb.Batch)
+		want := "simple string batch value"
+		simpleString.PutInBatch(batch, want)
+		err = db.WriteBatch(batch)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := simpleString.Get()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got != want {
+			t.Errorf("got string %q, want %q", got, want)
+		}
+
+		t.Run("overwrite", func(t *testing.T) {
+			batch := new(leveldb.Batch)
+			want := "overwritten string batch value"
+			simpleString.PutInBatch(batch, want)
+			err = db.WriteBatch(batch)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := simpleString.Get()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != want {
+				t.Errorf("got string %q, want %q", got, want)
+			}
+		})
+	})
+}

swarm/shed/internal/field_uint64.go

@@ -0,0 +1,91 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"encoding/binary"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type Uint64Field struct {
+	db  *DB
+	key []byte
+}
+
+func (db *DB) NewUint64Field(name string) (f Uint64Field, err error) {
+	key, err := db.schemaFieldKey(name, "uint64")
+	if err != nil {
+		return f, err
+	}
+	return Uint64Field{
+		db:  db,
+		key: key,
+	}, nil
+}
+
+func (f Uint64Field) Get() (val uint64, err error) {
+	b, err := f.db.Get(f.key)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return 0, nil
+		}
+		return 0, err
+	}
+	return binary.BigEndian.Uint64(b), nil
+}
+
+func (f Uint64Field) Put(val uint64) (err error) {
+	return f.db.Put(f.key, encodeUint64(val))
+}
+
+func (f Uint64Field) PutInBatch(batch *leveldb.Batch, val uint64) {
+	batch.Put(f.key, encodeUint64(val))
+}
+
+func (f Uint64Field) Inc() (val uint64, err error) {
+	val, err = f.Get()
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			val = 0
+		} else {
+			return 0, err
+		}
+	}
+	val++
+	return val, f.Put(val)
+}
+
+func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) {
+	val, err = f.Get()
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			val = 0
+		} else {
+			return 0, err
+		}
+	}
+	val++
+	f.PutInBatch(batch, val)
+	return val, nil
+}
+
+func encodeUint64(val uint64) (b []byte) {
+	b = make([]byte, 8)
+	binary.BigEndian.PutUint64(b, val)
+	return b
+}

swarm/shed/internal/field_uint64_test.go

@@ -0,0 +1,188 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+func TestUint64Field(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	counter, err := db.NewUint64Field("counter")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("get empty", func(t *testing.T) {
+		got, err := counter.Get()
+		if err != nil {
+			t.Fatal(err)
+		}
+		var want uint64 = 0
+		if got != want {
+			t.Errorf("got uint64 %v, want %v", got, want)
+		}
+	})
+
+	t.Run("put", func(t *testing.T) {
+		var want uint64 = 42
+		err = counter.Put(want)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := counter.Get()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got != want {
+			t.Errorf("got uint64 %v, want %v", got, want)
+		}
+
+		t.Run("overwrite", func(t *testing.T) {
+			var want uint64 = 84
+			err = counter.Put(want)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := counter.Get()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != want {
+				t.Errorf("got uint64 %v, want %v", got, want)
+			}
+		})
+	})
+
+	t.Run("put in batch", func(t *testing.T) {
+		batch := new(leveldb.Batch)
+		var want uint64 = 42
+		counter.PutInBatch(batch, want)
+		err = db.WriteBatch(batch)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := counter.Get()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got != want {
+			t.Errorf("got uint64 %v, want %v", got, want)
+		}
+
+		t.Run("overwrite", func(t *testing.T) {
+			batch := new(leveldb.Batch)
+			var want uint64 = 84
+			counter.PutInBatch(batch, want)
+			err = db.WriteBatch(batch)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := counter.Get()
+			if err != nil {
+				t.Fatal(err)
+			}
+			if got != want {
+				t.Errorf("got uint64 %v, want %v", got, want)
+			}
+		})
+	})
+}
+
+func TestUint64Field_Inc(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	counter, err := db.NewUint64Field("counter")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var want uint64 = 1
+	got, err := counter.Inc()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Errorf("got uint64 %v, want %v", got, want)
+	}
+
+	want = 2
+	got, err = counter.Inc()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Errorf("got uint64 %v, want %v", got, want)
+	}
+}
+
+func TestUint64Field_IncInBatch(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	counter, err := db.NewUint64Field("counter")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := new(leveldb.Batch)
+	var want uint64 = 1
+	got, err := counter.IncInBatch(batch)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Errorf("got uint64 %v, want %v", got, want)
+	}
+	err = db.WriteBatch(batch)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err = counter.Get()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Errorf("got uint64 %v, want %v", got, want)
+	}
+
+	batch2 := new(leveldb.Batch)
+	want = 2
+	got, err = counter.IncInBatch(batch2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Errorf("got uint64 %v, want %v", got, want)
+	}
+	err = db.WriteBatch(batch2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err = counter.Get()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != want {
+		t.Errorf("got uint64 %v, want %v", got, want)
+	}
+}

swarm/shed/internal/index.go

@@ -0,0 +1,204 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type IndexItem struct {
+	Hash            []byte
+	Data            []byte
+	AccessTimestamp int64
+	StoreTimestamp  int64
+}
+
+func (i IndexItem) Join(i2 IndexItem) (new IndexItem) {
+	if i.Hash == nil {
+		i.Hash = i2.Hash
+	}
+	if i.Data == nil {
+		i.Data = i2.Data
+	}
+	if i.AccessTimestamp == 0 {
+		i.AccessTimestamp = i2.AccessTimestamp
+	}
+	if i.StoreTimestamp == 0 {
+		i.StoreTimestamp = i2.StoreTimestamp
+	}
+	return i
+}
+
+type Index struct {
+	db              *DB
+	prefix          []byte
+	encodeKeyFunc   func(fields IndexItem) (key []byte, err error)
+	decodeKeyFunc   func(key []byte) (e IndexItem, err error)
+	encodeValueFunc func(fields IndexItem) (value []byte, err error)
+	decodeValueFunc func(value []byte) (e IndexItem, err error)
+}
+
+type IndexFuncs struct {
+	EncodeKey   func(fields IndexItem) (key []byte, err error)
+	DecodeKey   func(key []byte) (e IndexItem, err error)
+	EncodeValue func(fields IndexItem) (value []byte, err error)
+	DecodeValue func(value []byte) (e IndexItem, err error)
+}
+
+func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) {
+	id, err := db.schemaIndexID(name)
+	if err != nil {
+		return f, err
+	}
+	prefix := []byte{id}
+	return Index{
+		db:     db,
+		prefix: prefix,
+		encodeKeyFunc: func(e IndexItem) (key []byte, err error) {
+			key, err = funcs.EncodeKey(e)
+			if err != nil {
+				return nil, err
+			}
+			return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil
+		},
+		decodeKeyFunc: func(key []byte) (e IndexItem, err error) {
+			return funcs.DecodeKey(key[1:])
+		},
+		encodeValueFunc: funcs.EncodeValue,
+		decodeValueFunc: funcs.DecodeValue,
+	}, nil
+}
+
+func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) {
+	key, err := f.encodeKeyFunc(keyFields)
+	if err != nil {
+		return out, err
+	}
+	value, err := f.db.Get(key)
+	if err != nil {
+		return out, err
+	}
+	out, err = f.decodeValueFunc(value)
+	if err != nil {
+		return out, err
+	}
+	return out.Join(keyFields), nil
+}
+
+func (f Index) Put(i IndexItem) (err error) {
+	key, err := f.encodeKeyFunc(i)
+	if err != nil {
+		return err
+	}
+	value, err := f.encodeValueFunc(i)
+	if err != nil {
+		return err
+	}
+	return f.db.Put(key, value)
+}
+
+func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) {
+	key, err := f.encodeKeyFunc(i)
+	if err != nil {
+		return err
+	}
+	value, err := f.encodeValueFunc(i)
+	if err != nil {
+		return err
+	}
+	batch.Put(key, value)
+	return nil
+}
+
+func (f Index) Delete(keyFields IndexItem) (err error) {
+	key, err := f.encodeKeyFunc(keyFields)
+	if err != nil {
+		return err
+	}
+	return f.db.Delete(key)
+}
+
+func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error) {
+	key, err := f.encodeKeyFunc(keyFields)
+	if err != nil {
+		return err
+	}
+	batch.Delete(key)
+	return nil
+}
+
+type IterFunc func(item IndexItem) (stop bool, err error)
+
+func (f Index) IterateAll(fn IterFunc) (err error) {
+	it := f.db.NewIterator()
+	defer it.Release()
+
+	for ok := it.Seek(f.prefix); ok; ok = it.Next() {
+		key := it.Key()
+		if key[0] != f.prefix[0] {
+			break
+		}
+		keyIndexItem, err := f.decodeKeyFunc(key)
+		if err != nil {
+			return err
+		}
+		valueIndexItem, err := f.decodeValueFunc(it.Value())
+		if err != nil {
+			return err
+		}
+		stop, err := fn(keyIndexItem.Join(valueIndexItem))
+		if err != nil {
+			return err
+		}
+		if stop {
+			break
+		}
+	}
+	return it.Error()
+}
+
+func (f Index) IterateFrom(start IndexItem, fn IterFunc) (err error) {
+	startKey, err := f.encodeKeyFunc(start)
+	if err != nil {
+		return err
+	}
+	it := f.db.NewIterator()
+	defer it.Release()
+
+	for ok := it.Seek(startKey); ok; ok = it.Next() {
+		key := it.Key()
+		if key[0] != f.prefix[0] {
+			break
+		}
+		keyIndexItem, err := f.decodeKeyFunc(key)
+		if err != nil {
+			return err
+		}
+		valueIndexItem, err := f.decodeValueFunc(it.Value())
+		if err != nil {
+			return err
+		}
+		stop, err := fn(keyIndexItem.Join(valueIndexItem))
+		if err != nil {
+			return err
+		}
+		if stop {
+			break
+		}
+	}
+	return it.Error()
+}

swarm/shed/internal/index_test.go

@@ -0,0 +1,386 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+var retrievalIndexFuncs = IndexFuncs{
+	EncodeKey: func(fields IndexItem) (key []byte, err error) {
+		return fields.Hash, nil
+	},
+	DecodeKey: func(key []byte) (e IndexItem, err error) {
+		e.Hash = key
+		return e, nil
+	},
+	EncodeValue: func(fields IndexItem) (value []byte, err error) {
+		b := make([]byte, 8)
+		binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp))
+		value = append(b, fields.Data...)
+		return value, nil
+	},
+	DecodeValue: func(value []byte) (e IndexItem, err error) {
+		e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8]))
+		e.Data = value[8:]
+		return e, nil
+	},
+}
+
+func TestIndex(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	index, err := db.NewIndex("retrieval", retrievalIndexFuncs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("put", func(t *testing.T) {
+		want := IndexItem{
+			Hash:           []byte("put-hash"),
+			Data:           []byte("DATA"),
+			StoreTimestamp: time.Now().UTC().UnixNano(),
+		}
+
+		err = index.Put(want)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := index.Get(IndexItem{
+			Hash: want.Hash,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		checkIndexItem(t, got, want)
+
+		t.Run("overwrite", func(t *testing.T) {
+			want := IndexItem{
+				Hash:           []byte("put-hash"),
+				Data:           []byte("New DATA"),
+				StoreTimestamp: time.Now().UTC().UnixNano(),
+			}
+
+			err = index.Put(want)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := index.Get(IndexItem{
+				Hash: want.Hash,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			checkIndexItem(t, got, want)
+		})
+	})
+
+	t.Run("put in batch", func(t *testing.T) {
+		want := IndexItem{
+			Hash:           []byte("put-in-batch-hash"),
+			Data:           []byte("DATA"),
+			StoreTimestamp: time.Now().UTC().UnixNano(),
+		}
+
+		batch := new(leveldb.Batch)
+		index.PutInBatch(batch, want)
+		db.WriteBatch(batch)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := index.Get(IndexItem{
+			Hash: want.Hash,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		checkIndexItem(t, got, want)
+
+		t.Run("overwrite", func(t *testing.T) {
+			want := IndexItem{
+				Hash:           []byte("put-in-batch-hash"),
+				Data:           []byte("New DATA"),
+				StoreTimestamp: time.Now().UTC().UnixNano(),
+			}
+
+			batch := new(leveldb.Batch)
+			index.PutInBatch(batch, want)
+			db.WriteBatch(batch)
+			if err != nil {
+				t.Fatal(err)
+			}
+			got, err := index.Get(IndexItem{
+				Hash: want.Hash,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			checkIndexItem(t, got, want)
+		})
+	})
+
+	t.Run("delete", func(t *testing.T) {
+		want := IndexItem{
+			Hash:           []byte("delete-hash"),
+			Data:           []byte("DATA"),
+			StoreTimestamp: time.Now().UTC().UnixNano(),
+		}
+
+		err = index.Put(want)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := index.Get(IndexItem{
+			Hash: want.Hash,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		checkIndexItem(t, got, want)
+
+		err = index.Delete(IndexItem{
+			Hash: want.Hash,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		got, err = index.Get(IndexItem{
+			Hash: want.Hash,
+		})
+		if err != leveldb.ErrNotFound {
+			t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
+		}
+	})
+
+	t.Run("delete in batch", func(t *testing.T) {
+		want := IndexItem{
+			Hash:           []byte("delete-in-batch-hash"),
+			Data:           []byte("DATA"),
+			StoreTimestamp: time.Now().UTC().UnixNano(),
+		}
+
+		err = index.Put(want)
+		if err != nil {
+			t.Fatal(err)
+		}
+		got, err := index.Get(IndexItem{
+			Hash: want.Hash,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		checkIndexItem(t, got, want)
+
+		batch := new(leveldb.Batch)
+		index.DeleteInBatch(batch, IndexItem{
+			Hash: want.Hash,
+		})
+		err = db.WriteBatch(batch)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		got, err = index.Get(IndexItem{
+			Hash: want.Hash,
+		})
+		if err != leveldb.ErrNotFound {
+			t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
+		}
+	})
+}
+
+func TestIndex_iterate(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	index, err := db.NewIndex("retrieval", retrievalIndexFuncs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	items := []IndexItem{
+		{
+			Hash: []byte("iterate-hash-01"),
+			Data: []byte("data80"),
+		},
+		{
+			Hash: []byte("iterate-hash-03"),
+			Data: []byte("data22"),
+		},
+		{
+			Hash: []byte("iterate-hash-05"),
+			Data: []byte("data41"),
+		},
+		{
+			Hash: []byte("iterate-hash-02"),
+			Data: []byte("data84"),
+		},
+		{
+			Hash: []byte("iterate-hash-06"),
+			Data: []byte("data1"),
+		},
+	}
+	batch := new(leveldb.Batch)
+	for _, i := range items {
+		index.PutInBatch(batch, i)
+	}
+	err = db.WriteBatch(batch)
+	if err != nil {
+		t.Fatal(err)
+	}
+	item04 := IndexItem{
+		Hash: []byte("iterate-hash-04"),
+		Data: []byte("data0"),
+	}
+	err = index.Put(item04)
+	if err != nil {
+		t.Fatal(err)
+	}
+	items = append(items, item04)
+
+	sort.SliceStable(items, func(i, j int) bool {
+		return bytes.Compare(items[i].Hash, items[j].Hash) < 0
+	})
+
+	t.Run("all", func(t *testing.T) {
+		var i int
+		err := index.IterateAll(func(item IndexItem) (stop bool, err error) {
+			if i > len(items)-1 {
+				return true, fmt.Errorf("got unexpected index item: %#v", item)
+			}
+			want := items[i]
+			checkIndexItem(t, item, want)
+			i++
+			return false, nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("from", func(t *testing.T) {
+		startIndex := 2
+		i := startIndex
+		err := index.IterateFrom(items[startIndex], func(item IndexItem) (stop bool, err error) {
+			if i > len(items)-1 {
+				return true, fmt.Errorf("got unexpected index item: %#v", item)
+			}
+			want := items[i]
+			checkIndexItem(t, item, want)
+			i++
+			return false, nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("stop", func(t *testing.T) {
+		var i int
+		stopIndex := 3
+		var count int
+		err := index.IterateAll(func(item IndexItem) (stop bool, err error) {
+			if i > len(items)-1 {
+				return true, fmt.Errorf("got unexpected index item: %#v", item)
+			}
+			want := items[i]
+			checkIndexItem(t, item, want)
+			count++
+			if i == stopIndex {
+				return true, nil
+			}
+			i++
+			return false, nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		wantItemsCount := stopIndex + 1
+		if count != wantItemsCount {
+			t.Errorf("got %v items, expected %v", count, wantItemsCount)
+		}
+	})
+
+	t.Run("no overflow", func(t *testing.T) {
+		secondIndex, err := db.NewIndex("second-index", retrievalIndexFuncs)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		secondIndexItem := IndexItem{
+			Hash: []byte("iterate-hash-10"),
+			Data: []byte("data-second"),
+		}
+		err = secondIndex.Put(secondIndexItem)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var i int
+		err = index.IterateAll(func(item IndexItem) (stop bool, err error) {
+			if i > len(items)-1 {
+				return true, fmt.Errorf("got unexpected index item: %#v", item)
+			}
+			want := items[i]
+			checkIndexItem(t, item, want)
+			i++
+			return false, nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		i = 0
+		err = secondIndex.IterateAll(func(item IndexItem) (stop bool, err error) {
+			if i > 1 {
+				return true, fmt.Errorf("got unexpected index item: %#v", item)
+			}
+			checkIndexItem(t, item, secondIndexItem)
+			i++
+			return false, nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+}
+
+func checkIndexItem(t *testing.T, got, want IndexItem) {
+	t.Helper()
+
+	if !bytes.Equal(got.Hash, want.Hash) {
+		t.Errorf("got hash %q, expected %q", string(got.Hash), string(want.Hash))
+	}
+	if !bytes.Equal(got.Data, want.Data) {
+		t.Errorf("got data %q, expected %q", string(got.Data), string(want.Data))
+	}
+	if got.StoreTimestamp != want.StoreTimestamp {
+		t.Errorf("got store timestamp %v, expected %v", got.StoreTimestamp, want.StoreTimestamp)
+	}
+	if got.AccessTimestamp != want.AccessTimestamp {
+		t.Errorf("got access timestamp %v, expected %v", got.AccessTimestamp, want.AccessTimestamp)
+	}
+}

swarm/shed/internal/schema.go

@@ -0,0 +1,115 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+)
+
+var (
+	keySchema                = []byte{0}
+	keyPrefixFields     byte = 1
+	keyPrefixIndexStart byte = 2 // Q: or maybe 7, to have more space for potential specific perfixes
+)
+
+type schema struct {
+	Fields  map[string]fieldSpec `json:"fields"`
+	Indexes map[byte]indexSpec   `json:"indexes"`
+}
+
+type fieldSpec struct {
+	Type string `json:"type"`
+}
+
+type indexSpec struct {
+	Name string `json:"name"`
+}
+
+func (db *DB) schemaFieldKey(name, fieldType string) (key []byte, err error) {
+	if name == "" {
+		return nil, errors.New("filed name can not be blank")
+	}
+	if fieldType == "" {
+		return nil, errors.New("filed type can not be blank")
+	}
+	s, err := db.getSchema()
+	if err != nil {
+		return nil, err
+	}
+	var found bool
+	for n, f := range s.Fields {
+		if n == name {
+			if f.Type != fieldType {
+				return nil, fmt.Errorf("field %q of type %q stored as %q in db", name, fieldType, f.Type)
+			}
+			break
+		}
+	}
+	if !found {
+		s.Fields[name] = fieldSpec{
+			Type: fieldType,
+		}
+		err := db.putSchema(s)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return append([]byte{keyPrefixFields}, []byte(name)...), nil
+}
+
+func (db *DB) schemaIndexID(name string) (id byte, err error) {
+	if name == "" {
+		return 0, errors.New("index name can not be blank")
+	}
+	s, err := db.getSchema()
+	if err != nil {
+		return 0, err
+	}
+	nextID := keyPrefixIndexStart
+	for i, f := range s.Indexes {
+		if i >= nextID {
+			nextID = i + 1
+		}
+		if f.Name == name {
+			return i, nil
+		}
+	}
+	id = nextID
+	s.Indexes[id] = indexSpec{
+		Name: name,
+	}
+	return id, db.putSchema(s)
+}
+
+func (db *DB) getSchema() (s schema, err error) {
+	b, err := db.Get(keySchema)
+	if err != nil {
+		return s, err
+	}
+	err = json.Unmarshal(b, &s)
+	return s, err
+}
+
+func (db *DB) putSchema(s schema) (err error) {
+	b, err := json.Marshal(s)
+	if err != nil {
+		return err
+	}
+	return db.Put(keySchema, b)
+}

swarm/shed/internal/schema_test.go

@@ -0,0 +1,124 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package internal
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestSchema_schemaFieldKey(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	t.Run("empty name or type", func(t *testing.T) {
+		_, err := db.schemaFieldKey("", "")
+		if err == nil {
+			t.Errorf("error not returned, but expected")
+		}
+		_, err = db.schemaFieldKey("", "type")
+		if err == nil {
+			t.Errorf("error not returned, but expected")
+		}
+
+		_, err = db.schemaFieldKey("test", "")
+		if err == nil {
+			t.Errorf("error not returned, but expected")
+		}
+	})
+
+	t.Run("same field", func(t *testing.T) {
+		key1, err := db.schemaFieldKey("test", "undefined")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		key2, err := db.schemaFieldKey("test", "undefined")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !bytes.Equal(key1, key2) {
+			t.Errorf("schema keys for the same field name are not the same: %q, %q", string(key1), string(key2))
+		}
+	})
+
+	t.Run("different fields", func(t *testing.T) {
+		key1, err := db.schemaFieldKey("test1", "undefined")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		key2, err := db.schemaFieldKey("test2", "undefined")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if bytes.Equal(key1, key2) {
+			t.Error("schema keys for the same field name are the same, but must not be")
+		}
+	})
+
+	t.Run("same field name different types", func(t *testing.T) {
+		_, err := db.schemaFieldKey("the-field", "one-type")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		_, err = db.schemaFieldKey("the-field", "another-type")
+		if err == nil {
+			t.Errorf("error not returned, but expected")
+		}
+	})
+}
+
+func TestSchema_schemaIndexID(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	t.Run("same name", func(t *testing.T) {
+		id1, err := db.schemaIndexID("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		id2, err := db.schemaIndexID("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if id1 != id2 {
+			t.Errorf("schema keys for the same field name are not the same: %v, %v", id1, id2)
+		}
+	})
+
+	t.Run("different names", func(t *testing.T) {
+		id1, err := db.schemaIndexID("test1")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		id2, err := db.schemaIndexID("test2")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if id1 == id2 {
+			t.Error("schema ids for the same index name are the same, but must not be")
+		}
+	})
+}

swarm/shed/shed.go

@@ -0,0 +1,247 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package shed
+
+import (
+	"context"
+	"encoding/binary"
+	"time"
+
+	"github.com/ethereum/go-ethereum/swarm/shed/internal"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// DB is just an example for composing indexes.
+type DB struct {
+	db *internal.DB
+
+	// fields and indexes
+	schemaName     internal.StringField
+	sizeCounter    internal.Uint64Field
+	accessCounter  internal.Uint64Field
+	retrievalIndex internal.Index
+	accessIndex    internal.Index
+	gcIndex        internal.Index
+}
+
+func New(path string) (db *DB, err error) {
+	idb, err := internal.NewDB(path)
+	if err != nil {
+		return nil, err
+	}
+	db = &DB{
+		db: idb,
+	}
+	db.schemaName, err = idb.NewStringField("schema-name")
+	if err != nil {
+		return nil, err
+	}
+	db.sizeCounter, err = idb.NewUint64Field("size-counter")
+	if err != nil {
+		return nil, err
+	}
+	db.accessCounter, err = idb.NewUint64Field("access-counter")
+	if err != nil {
+		return nil, err
+	}
+	db.retrievalIndex, err = idb.NewIndex("Hash->StoreTimestamp|Data", internal.IndexFuncs{
+		EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
+			return fields.Hash, nil
+		},
+		DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
+			e.Hash = key
+			return e, nil
+		},
+		EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
+			b := make([]byte, 8)
+			binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp))
+			value = append(b, fields.Data...)
+			return value, nil
+		},
+		DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
+			e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8]))
+			e.Data = value[8:]
+			return e, nil
+		},
+	})
+	db.accessIndex, err = idb.NewIndex("Hash->AccessTimestamp", internal.IndexFuncs{
+		EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
+			return fields.Hash, nil
+		},
+		DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
+			e.Hash = key
+			return e, nil
+		},
+		EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
+			b := make([]byte, 8)
+			binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp))
+			return b, nil
+		},
+		DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
+			e.AccessTimestamp = int64(binary.BigEndian.Uint64(value))
+			return e, nil
+		},
+	})
+	db.gcIndex, err = idb.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", internal.IndexFuncs{
+		EncodeKey: func(fields internal.IndexItem) (key []byte, err error) {
+			b := make([]byte, 16, 16+len(fields.Hash))
+			binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp))
+			binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp))
+			key = append(b, fields.Hash...)
+			return key, nil
+		},
+		DecodeKey: func(key []byte) (e internal.IndexItem, err error) {
+			e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8]))
+			e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16]))
+			e.Hash = key[16:]
+			return e, nil
+		},
+		EncodeValue: func(fields internal.IndexItem) (value []byte, err error) {
+			return nil, nil
+		},
+		DecodeValue: func(value []byte) (e internal.IndexItem, err error) {
+			return e, nil
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	return db, nil
+}
+
+func (db *DB) Put(_ context.Context, ch storage.Chunk) (err error) {
+	return db.retrievalIndex.Put(internal.IndexItem{
+		Hash:           ch.Address(),
+		Data:           ch.Data(),
+		StoreTimestamp: time.Now().UTC().UnixNano(),
+	})
+}
+
+func (db *DB) Get(_ context.Context, ref storage.Address) (c storage.Chunk, err error) {
+	batch := new(leveldb.Batch)
+
+	item, err := db.retrievalIndex.Get(internal.IndexItem{
+		Hash: ref,
+	})
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return nil, storage.ErrChunkNotFound
+		}
+		return nil, err
+	}
+
+	accessItem, err := db.accessIndex.Get(internal.IndexItem{
+		Hash: ref,
+	})
+	switch err {
+	case nil:
+		err = db.gcIndex.DeleteInBatch(batch, internal.IndexItem{
+			Hash:            item.Hash,
+			StoreTimestamp:  accessItem.AccessTimestamp,
+			AccessTimestamp: item.StoreTimestamp,
+		})
+		if err != nil {
+			return nil, err
+		}
+	case leveldb.ErrNotFound:
+	default:
+		return nil, err
+	}
+
+	accessTimestamp := time.Now().UTC().UnixNano()
+
+	err = db.accessIndex.PutInBatch(batch, internal.IndexItem{
+		Hash:            ref,
+		AccessTimestamp: accessTimestamp,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = db.gcIndex.PutInBatch(batch, internal.IndexItem{
+		Hash:            item.Hash,
+		AccessTimestamp: accessTimestamp,
+		StoreTimestamp:  item.StoreTimestamp,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = db.db.WriteBatch(batch)
+	if err != nil {
+		return nil, err
+	}
+
+	return storage.NewChunk(item.Hash, item.Data), nil
+}
+
+func (db *DB) CollectGarbage() (err error) {
+	const maxTrashSize = 100
+	maxRounds := 10 // adbitrary number, needs to be calculated
+
+	for roundCount := 0; roundCount < maxRounds; roundCount++ {
+		var garbageCount int
+		trash := new(leveldb.Batch)
+		err = db.gcIndex.IterateAll(func(item internal.IndexItem) (stop bool, err error) {
+			err = db.retrievalIndex.DeleteInBatch(trash, item)
+			if err != nil {
+				return false, err
+			}
+			err = db.accessIndex.DeleteInBatch(trash, item)
+			if err != nil {
+				return false, err
+			}
+			err = db.gcIndex.DeleteInBatch(trash, item)
+			if err != nil {
+				return false, err
+			}
+			garbageCount++
+			if garbageCount >= maxTrashSize {
+				return true, nil
+			}
+			return false, nil
+		})
+		if err != nil {
+			return err
+		}
+		if garbageCount == 0 {
+			return nil
+		}
+		err = db.db.WriteBatch(trash)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (db *DB) GetSchema() (name string, err error) {
+	name, err = db.schemaName.Get()
+	if err == leveldb.ErrNotFound {
+		return "", nil
+	}
+	return name, err
+}
+
+func (db *DB) PutSchema(name string) (err error) {
+	return db.schemaName.Put(name)
+}
+
+func (db *DB) Close() {
+	db.db.Close()
+}