swarm/network, swarm/storage: validate default chunk size

swarm/bmt/bmt.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 )
 
 /*
@@ -318,7 +320,7 @@ func (h *Hasher) Sum(b []byte) (s []byte) {
 // with every full segment calls writeSection in a go routine
 func (h *Hasher) Write(b []byte) (int, error) {
 	l := len(b)
-	if l == 0 || l > 4096 {
+	if l == 0 || l > chunk.DefaultSize {
 		return 0, nil
 	}
 	t := h.getTree()

swarm/chunk/chunk.go

@@ -0,0 +1,5 @@
+package chunk
+
+const (
+	DefaultSize int64 = 4096
+)

swarm/network/stream/delivery.go

@@ -244,6 +244,11 @@ R:
 			continue R
 		default:
 		}
+		if len(req.SData) > chunk.DefaultSize {
+			log.Warn("received chunk is bigger than expected", "len", len(req.SData))
+			continue R
+		}
+
 		chunk.SData = req.SData
 		d.db.Put(context.TODO(), chunk)
 

swarm/storage/chunker.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/log"
 	"github.com/ethereum/go-ethereum/swarm/spancontext"
 	opentracing "github.com/opentracing/opentracing-go"
@@ -69,10 +70,6 @@ var (
 	errOperationTimedOut    = errors.New("operation timed out")
 )
 
-const (
-	DefaultChunkSize int64 = 4096
-)
-
 type ChunkerParams struct {
 	chunkSize int64
 	hashSize  int64
@@ -136,7 +133,7 @@ type TreeChunker struct {
 func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader {
 	jp := &JoinerParams{
 		ChunkerParams: ChunkerParams{
-			chunkSize: DefaultChunkSize,
+			chunkSize: chunk.DefaultSize,
 			hashSize:  int64(len(addr)),
 		},
 		addr:   addr,
@@ -156,7 +153,7 @@ func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (
 	tsp := &TreeSplitterParams{
 		SplitterParams: SplitterParams{
 			ChunkerParams: ChunkerParams{
-				chunkSize: DefaultChunkSize,
+				chunkSize: chunk.DefaultSize,
 				hashSize:  putter.RefSize(),
 			},
 			reader: data,

swarm/storage/hasherstore.go

@@ -22,6 +22,7 @@ import (
 	"sync"
 
 	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/storage/encryption"
 )
 
@@ -57,7 +58,7 @@ func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool)
 	refSize := int64(hashSize)
 	if toEncrypt {
 		refSize += encryption.KeyLength
-		chunkEncryption = newChunkEncryption(DefaultChunkSize, refSize)
+		chunkEncryption = newChunkEncryption(chunk.DefaultSize, refSize)
 	}
 
 	return &hasherStore{
@@ -190,9 +191,9 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp
 
 	// removing extra bytes which were just added for padding
 	length := ChunkData(decryptedSpan).Size()
-	for length > DefaultChunkSize {
+	for length > chunk.DefaultSize {
-		length = length + (DefaultChunkSize - 1)
+		length = length + (chunk.DefaultSize - 1)
-		length = length / DefaultChunkSize
+		length = length / chunk.DefaultSize
 		length *= h.refSize
 	}
 

swarm/storage/ldbstore_test.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/log"
 	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
 
@@ -184,7 +185,7 @@ func testIterator(t *testing.T, mock bool) {
 		t.Fatalf("init dbStore failed: %v", err)
 	}
 
-	chunks := GenerateRandomChunks(DefaultChunkSize, chunkcount)
+	chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount)
 
 	wg := &sync.WaitGroup{}
 	wg.Add(len(chunks))
@@ -294,7 +295,7 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) {
 
 	chunks := []*Chunk{}
 	for i := 0; i < n; i++ {
-		c := GenerateRandomChunk(DefaultChunkSize)
+		c := GenerateRandomChunk(chunk.DefaultSize)
 		chunks = append(chunks, c)
 		log.Trace("generate random chunk", "idx", i, "chunk", c)
 	}
@@ -344,7 +345,7 @@ func TestLDBStoreCollectGarbage(t *testing.T) {
 
 	chunks := []*Chunk{}
 	for i := 0; i < n; i++ {
-		c := GenerateRandomChunk(DefaultChunkSize)
+		c := GenerateRandomChunk(chunk.DefaultSize)
 		chunks = append(chunks, c)
 		log.Trace("generate random chunk", "idx", i, "chunk", c)
 	}
@@ -398,7 +399,7 @@ func TestLDBStoreAddRemove(t *testing.T) {
 
 	chunks := []*Chunk{}
 	for i := 0; i < n; i++ {
-		c := GenerateRandomChunk(DefaultChunkSize)
+		c := GenerateRandomChunk(chunk.DefaultSize)
 		chunks = append(chunks, c)
 		log.Trace("generate random chunk", "idx", i, "chunk", c)
 	}
@@ -460,7 +461,7 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) {
 
 	chunks := []*Chunk{}
 	for i := 0; i < capacity; i++ {
-		c := GenerateRandomChunk(DefaultChunkSize)
+		c := GenerateRandomChunk(chunk.DefaultSize)
 		chunks = append(chunks, c)
 		log.Trace("generate random chunk", "idx", i, "chunk", c)
 	}

swarm/storage/localstore_test.go

@@ -20,6 +20,8 @@ import (
 	"io/ioutil"
 	"os"
 	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 )
 
 var (
@@ -61,7 +63,7 @@ func TestValidator(t *testing.T) {
 	// add content address validator and check puts
 	// bad should fail, good should pass
 	store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc))
-	chunks = GenerateRandomChunks(DefaultChunkSize, 2)
+	chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
 	goodChunk = chunks[0]
 	badChunk = chunks[1]
 	copy(badChunk.SData, goodChunk.SData)
@@ -79,7 +81,7 @@ func TestValidator(t *testing.T) {
 	var negV boolTestValidator
 	store.Validators = append(store.Validators, negV)
 
-	chunks = GenerateRandomChunks(DefaultChunkSize, 2)
+	chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
 	goodChunk = chunks[0]
 	badChunk = chunks[1]
 	copy(badChunk.SData, goodChunk.SData)
@@ -97,7 +99,7 @@ func TestValidator(t *testing.T) {
 	var posV boolTestValidator = true
 	store.Validators = append(store.Validators, posV)
 
-	chunks = GenerateRandomChunks(DefaultChunkSize, 2)
+	chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
 	goodChunk = chunks[0]
 	badChunk = chunks[1]
 	copy(badChunk.SData, goodChunk.SData)

swarm/storage/mru/resource_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/contracts/ens"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
@@ -783,7 +784,7 @@ func TestValidatorInStore(t *testing.T) {
 	store.Validators = append(store.Validators, rh)
 
 	// create content addressed chunks, one good, one faulty
-	chunks := storage.GenerateRandomChunks(storage.DefaultChunkSize, 2)
+	chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2)
 	goodChunk := chunks[0]
 	badChunk := chunks[1]
 	badChunk.SData = goodChunk.SData

swarm/storage/pyramid.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 	"github.com/ethereum/go-ethereum/swarm/log"
 )
 
@@ -101,11 +102,11 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get
 	New chunks to store are store using the putter which the caller provides.
 */
 func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
-	return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, DefaultChunkSize)).Split(ctx)
+	return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx)
 }
 
 func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
-	return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, DefaultChunkSize)).Append(ctx)
+	return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx)
 }
 
 // Entry to create a tree node

swarm/storage/types.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/swarm/bmt"
+	"github.com/ethereum/go-ethereum/swarm/chunk"
 )
 
 const MaxPO = 16
@@ -230,8 +231,8 @@ func GenerateRandomChunk(dataSize int64) *Chunk {
 func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk) {
 	var i int
 	hasher := MakeHashFunc(DefaultHash)()
-	if dataSize > DefaultChunkSize {
+	if dataSize > chunk.DefaultSize {
-		dataSize = DefaultChunkSize
+		dataSize = chunk.DefaultSize
 	}
 
 	for i = 0; i < count; i++ {