swarm/storage: use validators to cleanup LevelDB

swarm/storage/ldbstore.go

@@ -37,7 +37,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/rlp"
-	ch "github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/log"
 	"github.com/ethereum/go-ethereum/swarm/storage/mock"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -61,7 +60,7 @@ var (
 	keyDataIdx     = []byte{4}
 	keyData        = byte(6)
 	keyDistanceCnt = byte(7)
-	keySchema      = byte(8)
+	keySchema      = []byte{8}
 )
 
 var (
@@ -419,7 +418,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) {
 	}
 }
 
-func (s *LDBStore) Cleanup() {
+func (s *LDBStore) Cleanup(f func(*chunk) bool) {
 	//Iterates over the database and checks that there are no chunks bigger than 4kb
 	var errorsFound, removed, total int
 
@@ -472,7 +471,8 @@ func (s *LDBStore) Cleanup() {
 		cs := int64(binary.LittleEndian.Uint64(c.sdata[:8]))
 		log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs)
 
-		if len(c.sdata) > ch.DefaultSize+8 {
+		// if chunk is to be removed
+		if f(c) {
 			log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs)
 			s.delete(index.Idx, getIndexKey(key[1:]), po)
 			removed++

swarm/storage/localstore.go

@@ -199,7 +199,17 @@ func (ls *LocalStore) Migrate() error {
 		if schema == "" {
 			log.Debug("running migrations for", "schema", schema, "runtime-schema", CurrentDbSchema)
 
-			ls.DbStore.Cleanup()
+			cleanupFunc := func(c *chunk) bool {
+				valid := false
+				for _, v := range ls.Validators {
+					if valid = v.Validate(c.Address(), c.Data()); valid {
+						break
+					}
+				}
+				return valid
+			}
+
+			ls.DbStore.Cleanup(cleanupFunc)
 
 			err := ls.DbStore.PutSchema(DbSchemaHive)
 			if err != nil {

swarm/swarm.go

@@ -169,11 +169,6 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
 		return nil, err
 	}
 
-	err = lstore.Migrate()
-	if err != nil {
-		return nil, err
-	}
-
 	self.netStore, err = storage.NewNetStore(lstore, nil)
 	if err != nil {
 		return nil, err
@@ -207,6 +202,11 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
 		resourceHandler,
 	}
 
+	err = lstore.Migrate()
+	if err != nil {
+		return nil, err
+	}
+
 	log.Debug("Setup local storage")
 
 	self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run)