diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index 19ff60de90..2573f42072 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -48,7 +48,7 @@ type DB struct { // fields schemaName shed.StringField - // this flag is for banchmarking two types of retrieval indexes + // this flag is for benchmarking two types of retrieval indexes // - single retrieval composite index retrievalCompositeIndex // - two separated indexes for data and access time // - retrievalDataIndex @@ -74,7 +74,7 @@ type DB struct { // Options struct holds optional parameters for configuring DB. type Options struct { - // UseRetrievalCompositeIndex option is for banchmarking + // UseRetrievalCompositeIndex option is for benchmarking // two types of retrieval indexes: // - single retrieval composite index retrievalCompositeIndex // - two separated indexes for data and access time @@ -305,6 +305,9 @@ func New(path string, baseKey []byte, o *Options) (db *DB, err error) { return e, nil }, }) + if err != nil { + return nil, err + } // count number of elements in garbage collection index var gcSize int64 db.gcIndex.IterateAll(func(_ shed.IndexItem) (stop bool, err error) { @@ -312,9 +315,6 @@ func New(path string, baseKey []byte, o *Options) (db *DB, err error) { return false, nil }) atomic.AddInt64(&db.gcSize, gcSize) - if err != nil { - return nil, err - } return db, nil } diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index 7e276ad269..a62af8ac6a 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -17,10 +17,14 @@ package localstore import ( - "crypto/rand" + "bytes" + "context" "io/ioutil" + "math/rand" "os" + "strconv" "testing" + "time" ch "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage" @@ -56,13 +60,96 @@ func TestDB_useRetrievalCompositeIndex(t *testing.T) { }) } +// BenchmarkNew measures the time that New function +// needs to initialize and count the number of key/value +// pairs in GC index. +// This benchmark generates a number of chunks, uploads them, +// sets them to synced state for them to enter the GC index, +// and measures the execution time of New function by creating +// new databases with the same data directory. +// +// This benchmark takes significant amount of time. +// +// Measurements on MacBook Pro (Retina, 15-inch, Mid 2014) show +// that New function executes around 1s for database with 1M chunks. +// +// # go test -benchmem -run=none github.com/ethereum/go-ethereum/swarm/storage/localstore -bench BenchmarkNew -v -timeout 20m +// goos: darwin +// goarch: amd64 +// pkg: github.com/ethereum/go-ethereum/swarm/storage/localstore +// BenchmarkNew/1000-8 200 12020231 ns/op 9556077 B/op 9999 allocs/op +// BenchmarkNew/10000-8 100 15475883 ns/op 10493071 B/op 7781 allocs/op +// BenchmarkNew/100000-8 20 64046466 ns/op 17823841 B/op 23375 allocs/op +// BenchmarkNew/1000000-8 1 1011464203 ns/op 51024688 B/op 310599 allocs/op +// PASS +func BenchmarkNew(b *testing.B) { + if testing.Short() { + b.Skip("skipping benchmark in short mode") + } + for _, count := range []int{ + 1000, + 10000, + 100000, + 1000000, + } { + b.Run(strconv.Itoa(count), func(b *testing.B) { + dir, err := ioutil.TempDir("", "localstore-new-benchmark") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(dir) + baseKey := make([]byte, 32) + if _, err := rand.Read(baseKey); err != nil { + b.Fatal(err) + } + db, err := New(dir, baseKey, nil) + if err != nil { + b.Fatal(err) + } + uploader := db.Accessor(ModeUpload) + syncer := db.Accessor(ModeSynced) + ctx := context.Background() + for i := 0; i < count; i++ { + chunk := generateFakeRandomChunk() + err := uploader.Put(ctx, chunk) + if err != nil { + b.Fatal(err) + } + err = syncer.Put(ctx, chunk) + if err != nil { + b.Fatal(err) + } + } + err = db.Close() + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + + for n := 0; n < b.N; n++ { + b.StartTimer() + db, err := New(dir, baseKey, nil) + b.StopTimer() + + if err != nil { + b.Fatal(err) + } + err = db.Close() + if err != nil { + b.Fatal(err) + } + } + }) + } +} + // newTestDB is a helper function that constructs a // temporary database and returns a cleanup function that must // be called to remove the data. -func newTestDB(t *testing.T, o *Options) (db *DB, cleanupFunc func()) { +func newTestDB(t testing.TB, o *Options) (db *DB, cleanupFunc func()) { t.Helper() - dir, err := ioutil.TempDir("", "shed-test") + dir, err := ioutil.TempDir("", "localstore-test") if err != nil { t.Fatal(err) } @@ -86,6 +173,42 @@ func newTestDB(t *testing.T, o *Options) (db *DB, cleanupFunc func()) { return db, cleanupFunc } +// generateRandomChunk generates a valid Chunk with +// data size of default chunk size. func generateRandomChunk() storage.Chunk { return storage.GenerateRandomChunk(ch.DefaultSize) } + +func init() { + // needed for generateFakeRandomChunk + rand.Seed(time.Now().UnixNano()) +} + +// generateFakeRandomChunk generates a Chunk that is not +// valid, but it contains a random key and a random value. +// This function is faster then storage.GenerateRandomChunk +// which generates a valid chunk. +// Some tests in this package do not need valid chunks, just +// random data, and their execution time can be decreased +// using this function. +func generateFakeRandomChunk() storage.Chunk { + data := make([]byte, ch.DefaultSize) + rand.Read(data) + key := make([]byte, 32) + rand.Read(key) + return storage.NewChunk(key, data) +} + +// TestGenerateFakeRandomChunk validates that +// generateFakeRandomChunk returns random data by comparing +// two generated chunks. +func TestGenerateFakeRandomChunk(t *testing.T) { + c1 := generateFakeRandomChunk() + c2 := generateFakeRandomChunk() + if bytes.Equal(c1.Address(), c2.Address()) { + t.Error("fake chunks addresses do not differ") + } + if bytes.Equal(c1.Data(), c2.Data()) { + t.Error("fake chunks data bytes do not differ") + } +}