diff --git a/swarm/storage/memstore_lrucache.go b/swarm/storage/memstore.go similarity index 98% rename from swarm/storage/memstore_lrucache.go rename to swarm/storage/memstore.go index 70bd709728..7266bc92be 100644 --- a/swarm/storage/memstore_lrucache.go +++ b/swarm/storage/memstore.go @@ -28,7 +28,7 @@ import ( type MemStore struct { cache *lru.Cache requests *lru.Cache - mu sync.Mutex + mu sync.RWMutex disabled bool } @@ -73,8 +73,8 @@ func (m *MemStore) Get(key Key) (*Chunk, error) { return nil, ErrChunkNotFound } - m.mu.Lock() - defer m.mu.Unlock() + m.mu.RLock() + defer m.mu.RUnlock() r, ok := m.requests.Get(string(key)) // it is a request diff --git a/swarm/storage/memstore_lrucache_test.go b/swarm/storage/memstore_lrucache_test.go deleted file mode 100644 index 8765bed40e..0000000000 --- a/swarm/storage/memstore_lrucache_test.go +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package storage - -import ( - "crypto/rand" - "encoding/binary" - "io/ioutil" - "os" - "sync" - "testing" - - "github.com/ethereum/go-ethereum/log" -) - -func newLDBStore(t *testing.T) (*LDBStore, func()) { - dir, err := ioutil.TempDir("", "bzz-storage-test") - if err != nil { - t.Fatal(err) - } - log.Trace("memstore.tempdir", "dir", dir) - - ldbparams := NewLDBStoreParams(NewDefaultStoreParams(), dir) - db, err := NewLDBStore(ldbparams) - if err != nil { - t.Fatal(err) - } - - cleanup := func() { - db.Close() - err := os.RemoveAll(dir) - if err != nil { - t.Fatal(err) - } - } - - return db, cleanup -} - -func TestMemStoreAndLDBStore(t *testing.T) { - ldb, cleanup := newLDBStore(t) - ldb.setCapacity(4000) - defer cleanup() - - cacheCap := 200 - requestsCap := 200 - memStore := NewMemStore(NewStoreParams(4000, 200, 200, nil, nil), nil) - - tests := []struct { - n int // number of chunks to push to memStore - chunkSize uint64 // size of chunk (by default in Swarm - 4096) - request bool // whether or not to set the ReqC channel on the random chunks - }{ - { - n: 1, - chunkSize: 4096, - request: false, - }, - { - n: 201, - chunkSize: 4096, - request: false, - }, - { - n: 501, - chunkSize: 4096, - request: false, - }, - { - n: 3100, - chunkSize: 4096, - request: false, - }, - { - n: 100, - chunkSize: 4096, - request: true, - }, - } - - for i, tt := range tests { - log.Info("running test", "idx", i, "tt", tt) - var chunks []*Chunk - - for i := 0; i < tt.n; i++ { - var c *Chunk - if tt.request { - c = NewRandomRequestChunk(tt.chunkSize) - } else { - c = NewRandomChunk(tt.chunkSize) - } - - chunks = append(chunks, c) - } - - for i := 0; i < tt.n; i++ { - go ldb.Put(chunks[i]) - memStore.Put(chunks[i]) - - if got := memStore.cache.Len(); got > cacheCap { - t.Fatalf("expected to get cache capacity less than %v, but got %v", cacheCap, got) - } - - if got := memStore.requests.Len(); got > requestsCap { - t.Fatalf("expected to get requests capacity less than %v, but got %v", requestsCap, got) - } - } - - for i := 0; i < tt.n; i++ { - _, err := memStore.Get(chunks[i].Key) - if err != nil { - if err == ErrChunkNotFound { - _, err := ldb.Get(chunks[i].Key) - if err != nil { - t.Fatalf("couldn't get chunk %v from ldb, got error: %v", i, err) - } - } else { - t.Fatalf("got error from memstore: %v", err) - } - } - } - - // wait for all chunks to be stored before ending the test are cleaning up - for i := 0; i < tt.n; i++ { - <-chunks[i].dbStoredC - } - } -} - -func NewRandomChunk(chunkSize uint64) *Chunk { - c := &Chunk{ - Key: make([]byte, 32), - ReqC: nil, - SData: make([]byte, chunkSize+8), // SData should be chunkSize + 8 bytes reserved for length - dbStoredC: make(chan bool), - dbStoredMu: &sync.Mutex{}, - } - - rand.Read(c.SData) - - binary.LittleEndian.PutUint64(c.SData[:8], chunkSize) - - hasher := MakeHashFunc(SHA3Hash)() - hasher.Write(c.SData) - copy(c.Key, hasher.Sum(nil)) - - return c -} - -func NewRandomRequestChunk(chunkSize uint64) *Chunk { - c := NewRandomChunk(chunkSize) - c.ReqC = make(chan bool) - - return c -} diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index f2e2f81a3c..5dc3ae5f86 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -16,7 +16,16 @@ package storage -import "testing" +import ( + "crypto/rand" + "encoding/binary" + "io/ioutil" + "os" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/log" +) func newTestMemStore() *MemStore { storeparams := NewDefaultStoreParams() @@ -96,3 +105,144 @@ func BenchmarkMemStoreGet_1_5k(b *testing.B) { func BenchmarkMemStoreGet_8_5k(b *testing.B) { benchmarkMemStoreGet(5000, 8, 4096, b) } + +func newLDBStore(t *testing.T) (*LDBStore, func()) { + dir, err := ioutil.TempDir("", "bzz-storage-test") + if err != nil { + t.Fatal(err) + } + log.Trace("memstore.tempdir", "dir", dir) + + ldbparams := NewLDBStoreParams(NewDefaultStoreParams(), dir) + db, err := NewLDBStore(ldbparams) + if err != nil { + t.Fatal(err) + } + + cleanup := func() { + db.Close() + err := os.RemoveAll(dir) + if err != nil { + t.Fatal(err) + } + } + + return db, cleanup +} + +func TestMemStoreAndLDBStore(t *testing.T) { + ldb, cleanup := newLDBStore(t) + ldb.setCapacity(4000) + defer cleanup() + + cacheCap := 200 + requestsCap := 200 + memStore := NewMemStore(NewStoreParams(4000, 200, 200, nil, nil), nil) + + tests := []struct { + n int // number of chunks to push to memStore + chunkSize uint64 // size of chunk (by default in Swarm - 4096) + request bool // whether or not to set the ReqC channel on the random chunks + }{ + { + n: 1, + chunkSize: 4096, + request: false, + }, + { + n: 201, + chunkSize: 4096, + request: false, + }, + { + n: 501, + chunkSize: 4096, + request: false, + }, + { + n: 3100, + chunkSize: 4096, + request: false, + }, + { + n: 100, + chunkSize: 4096, + request: true, + }, + } + + for i, tt := range tests { + log.Info("running test", "idx", i, "tt", tt) + var chunks []*Chunk + + for i := 0; i < tt.n; i++ { + var c *Chunk + if tt.request { + c = NewRandomRequestChunk(tt.chunkSize) + } else { + c = NewRandomChunk(tt.chunkSize) + } + + chunks = append(chunks, c) + } + + for i := 0; i < tt.n; i++ { + go ldb.Put(chunks[i]) + memStore.Put(chunks[i]) + + if got := memStore.cache.Len(); got > cacheCap { + t.Fatalf("expected to get cache capacity less than %v, but got %v", cacheCap, got) + } + + if got := memStore.requests.Len(); got > requestsCap { + t.Fatalf("expected to get requests capacity less than %v, but got %v", requestsCap, got) + } + } + + for i := 0; i < tt.n; i++ { + _, err := memStore.Get(chunks[i].Key) + if err != nil { + if err == ErrChunkNotFound { + _, err := ldb.Get(chunks[i].Key) + if err != nil { + t.Fatalf("couldn't get chunk %v from ldb, got error: %v", i, err) + } + } else { + t.Fatalf("got error from memstore: %v", err) + } + } + } + + // wait for all chunks to be stored before ending the test are cleaning up + for i := 0; i < tt.n; i++ { + <-chunks[i].dbStoredC + } + } +} + +func NewRandomChunk(chunkSize uint64) *Chunk { + c := &Chunk{ + Key: make([]byte, 32), + ReqC: nil, + SData: make([]byte, chunkSize+8), // SData should be chunkSize + 8 bytes reserved for length + dbStoredC: make(chan bool), + dbStoredMu: &sync.Mutex{}, + } + + rand.Read(c.SData) + + binary.LittleEndian.PutUint64(c.SData[:8], chunkSize) + + hasher := MakeHashFunc(SHA3Hash)() + hasher.Write(c.SData) + copy(c.Key, hasher.Sum(nil)) + + return c +} + +func NewRandomRequestChunk(chunkSize uint64) *Chunk { + c := NewRandomChunk(chunkSize) + c.ReqC = make(chan bool) + + return c +}