From df27fe73918b58ed5586339e067cd1bae5e124b6 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 25 Oct 2017 18:35:58 +0200 Subject: [PATCH 1/6] swarm/storage: Fix chunk overwrite on store restart - Upon re-open of store, last chunk got overwritten - Underlying db never closed from dpa and localstore - LocalDPA used different hash algo for verification than chunk hasher --- swarm/storage/dbstore.go | 9 +++++++++ swarm/storage/dpa.go | 8 ++++++-- swarm/storage/localstore.go | 4 +++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 46a5c16ccc..4dec144755 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -95,10 +95,19 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s * data, _ := s.db.Get(keyEntryCnt) s.entryCnt = BytesToU64(data) + if len(data) > 0 { + s.entryCnt++ + } data, _ = s.db.Get(keyAccessCnt) s.accessCnt = BytesToU64(data) + if len(data) > 0 { + s.accessCnt++ + } data, _ = s.db.Get(keyDataIdx) s.dataIdx = BytesToU64(data) + if len(data) > 0 { + s.dataIdx++ + } s.gcPos, _ = s.db.Get(keyGCPos) if s.gcPos == nil { s.gcPos = s.gcStartPos diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 44a2669f12..23fda3abf0 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -63,9 +63,12 @@ type DPA struct { } // for testing locally -func NewLocalDPA(datadir string) (*DPA, error) { +func NewLocalDPA(datadir string, hashalgorithm string) (*DPA, error) { - hash := MakeHashFunc("SHA256") + if hashalgorithm == "" { + hashalgorithm = "SHA3" + } + hash := MakeHashFunc(hashalgorithm) dbStore, err := NewDbStore(datadir, hash, singletonSwarmDbCapacity, 0) if err != nil { @@ -116,6 +119,7 @@ func (self *DPA) Start() { func (self *DPA) Stop() { self.lock.Lock() + self.Close() defer self.lock.Unlock() if !self.running { return diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index b442e6cc54..bf9eeb2e77 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -74,4 +74,6 @@ func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) { } // Close local store -func (self *LocalStore) Close() {} +func (self *LocalStore) Close() { + self.DbStore.Close() +} From 4a91e70031220b8f2786a5dcbd661d955be7a542 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 26 Oct 2017 13:19:05 +0200 Subject: [PATCH 2/6] swarm: Add missing param for NewLocalDPA --- swarm/swarm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/swarm.go b/swarm/swarm.go index 9db15325ae..ab447d9240 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -320,7 +320,7 @@ func NewLocalSwarm(datadir, port string) (self *Swarm, err error) { } config.Port = port - dpa, err := storage.NewLocalDPA(datadir) + dpa, err := storage.NewLocalDPA(datadir, "") if err != nil { return } From fb4f106ef76756e7287a70b12a8d8c9bb69a6148 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 26 Oct 2017 14:48:30 +0200 Subject: [PATCH 3/6] swarm/api, swarm/fuse: NewLocalDPA missing params --- swarm/api/api_test.go | 2 +- swarm/fuse/swarmfs_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index f9caed27f5..7ce24478df 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -36,7 +36,7 @@ func testApi(t *testing.T, f func(*Api)) { } os.RemoveAll(datadir) defer os.RemoveAll(datadir) - dpa, err := storage.NewLocalDPA(datadir) + dpa, err := storage.NewLocalDPA(datadir, "") if err != nil { return } diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 93f1d4c2fd..510ddd1247 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -810,7 +810,7 @@ func TestFUSE(t *testing.T) { } os.RemoveAll(datadir) - dpa, err := storage.NewLocalDPA(datadir) + dpa, err := storage.NewLocalDPA(datadir, "") if err != nil { t.Fatal(err) } From e833eb86eb46e5fa7e0226a7b1fb8c460b813cfb Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 10 Nov 2017 16:41:53 +0100 Subject: [PATCH 4/6] swarm/storage: Move defer lock unlock before db close --- swarm/storage/dpa.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 23fda3abf0..fb9dde3fe4 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -119,8 +119,8 @@ func (self *DPA) Start() { func (self *DPA) Stop() { self.lock.Lock() - self.Close() defer self.lock.Unlock() + self.Close() if !self.running { return } From 4a923691e74b34e058aeb74fc767e1400502758f Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 11 Nov 2017 17:35:53 +0100 Subject: [PATCH 5/6] swarm/storage: Add commend on count increment on open Rebase on master --- swarm/storage/dbstore.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 4dec144755..2c4ec25cf7 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -93,6 +93,8 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s * s.gcStartPos[0] = kpIndex s.gcArray = make([]*gcItem, gcArraySize) + // the database closes with pointer to last entry stored + // we increment upon open so that the last entry doesn't get overwritten data, _ := s.db.Get(keyEntryCnt) s.entryCnt = BytesToU64(data) if len(data) > 0 { From fe2f4ba2d61cd2ed96e34ac8b04526a70ad163cd Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 28 Nov 2017 19:10:41 +0100 Subject: [PATCH 6/6] swarm/storage: Add parametric store limits for localdpa + chunksize const --- swarm/storage/dpa.go | 15 +++++++++++---- swarm/storage/types.go | 4 ++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index fb9dde3fe4..adbf915335 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -63,20 +63,27 @@ type DPA struct { } // for testing locally -func NewLocalDPA(datadir string, hashalgorithm string) (*DPA, error) { +func NewLocalDPA(datadir string, hashalgorithm string, dbcapacity uint64, memcapacity uint) (*DPA, error) { if hashalgorithm == "" { hashalgorithm = "SHA3" } hash := MakeHashFunc(hashalgorithm) - dbStore, err := NewDbStore(datadir, hash, singletonSwarmDbCapacity, 0) + if dbcapacity == 0 { + dbcapacity = uint64(singletonSwarmDbCapacity) + } + if memcapacity == 0 { + memcapacity = uint(singletonSwarmCacheCapacity) + } + + log.Debug("LocalDPA create", "dbcap", dbcapacity, "memcap", memcapacity) + dbStore, err := NewDbStore(datadir, hash, dbcapacity, 0) if err != nil { return nil, err } - return NewDPA(&LocalStore{ - NewMemStore(dbStore, singletonSwarmCacheCapacity), + NewMemStore(dbStore, memcapacity), dbStore, }, NewChunkerParams()), nil } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index d35f1f9294..860659020a 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -29,6 +29,10 @@ import ( "github.com/ethereum/go-ethereum/crypto/sha3" ) +const ( + CHUNKSIZE = 4096 +) + type Hasher func() hash.Hash type SwarmHasher func() SwarmHash