forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-19 10:22:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

swarm/network, swarm/storage: validate default chunk size

Browse source
This commit is contained in:
Anton Evangelatov 2018-08-14 10:12:07 +02:00
parent e07e507d1a
commit 395c1a2085
10 changed files with 40 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
swarm/bmt/bmt.go
View file

@ -23,6 +23,8 @@ import (
"strings"
"sync"
"sync/atomic"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
/*
@ -318,7 +320,7 @@ func (h *Hasher) Sum(b []byte) (s []byte) {
// with every full segment calls writeSection in a go routine
func (h *Hasher) Write(b []byte) (int, error) {
l := len(b)
if l == 0 || l > 4096 {
if l == 0 || l > chunk.DefaultSize {
return 0, nil
}
t := h.getTree()

5
swarm/chunk/chunk.go Normal file
View file

@ -0,0 +1,5 @@
package chunk
const (
DefaultSize int64 = 4096
)

5
swarm/network/stream/delivery.go
View file

@ -244,6 +244,11 @@ R:
continue R
default:
}
if len(req.SData) > chunk.DefaultSize {
log.Warn("received chunk is bigger than expected", "len", len(req.SData))
continue R
}
chunk.SData = req.SData
d.db.Put(context.TODO(), chunk)

9
swarm/storage/chunker.go
View file

@ -25,6 +25,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/spancontext"
opentracing "github.com/opentracing/opentracing-go"
@ -69,10 +70,6 @@ var (
errOperationTimedOut = errors.New("operation timed out")
)
const (
DefaultChunkSize int64 = 4096
)
type ChunkerParams struct {
chunkSize int64
hashSize int64
@ -136,7 +133,7 @@ type TreeChunker struct {
func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader {
jp := &JoinerParams{
ChunkerParams: ChunkerParams{
chunkSize: DefaultChunkSize,
chunkSize: chunk.DefaultSize,
hashSize: int64(len(addr)),
},
addr: addr,
@ -156,7 +153,7 @@ func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (
tsp := &TreeSplitterParams{
SplitterParams: SplitterParams{
ChunkerParams: ChunkerParams{
chunkSize: DefaultChunkSize,
chunkSize: chunk.DefaultSize,
hashSize: putter.RefSize(),
},
reader: data,

9
swarm/storage/hasherstore.go
View file

@ -22,6 +22,7 @@ import (
"sync"
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/storage/encryption"
)
@ -57,7 +58,7 @@ func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool)
refSize := int64(hashSize)
if toEncrypt {
refSize += encryption.KeyLength
chunkEncryption = newChunkEncryption(DefaultChunkSize, refSize)
chunkEncryption = newChunkEncryption(chunk.DefaultSize, refSize)
}
return &hasherStore{
@ -190,9 +191,9 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp
// removing extra bytes which were just added for padding
length := ChunkData(decryptedSpan).Size()
for length > DefaultChunkSize {
length = length + (DefaultChunkSize - 1)
length = length / DefaultChunkSize
for length > chunk.DefaultSize {
length = length + (chunk.DefaultSize - 1)
length = length / chunk.DefaultSize
length *= h.refSize
}

11
swarm/storage/ldbstore_test.go
View file

@ -27,6 +27,7 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
@ -184,7 +185,7 @@ func testIterator(t *testing.T, mock bool) {
t.Fatalf("init dbStore failed: %v", err)
}
chunks := GenerateRandomChunks(DefaultChunkSize, chunkcount)
chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount)
wg := &sync.WaitGroup{}
wg.Add(len(chunks))
@ -294,7 +295,7 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) {
chunks := []*Chunk{}
for i := 0; i < n; i++ {
c := GenerateRandomChunk(DefaultChunkSize)
c := GenerateRandomChunk(chunk.DefaultSize)
chunks = append(chunks, c)
log.Trace("generate random chunk", "idx", i, "chunk", c)
}
@ -344,7 +345,7 @@ func TestLDBStoreCollectGarbage(t *testing.T) {
chunks := []*Chunk{}
for i := 0; i < n; i++ {
c := GenerateRandomChunk(DefaultChunkSize)
c := GenerateRandomChunk(chunk.DefaultSize)
chunks = append(chunks, c)
log.Trace("generate random chunk", "idx", i, "chunk", c)
}
@ -398,7 +399,7 @@ func TestLDBStoreAddRemove(t *testing.T) {
chunks := []*Chunk{}
for i := 0; i < n; i++ {
c := GenerateRandomChunk(DefaultChunkSize)
c := GenerateRandomChunk(chunk.DefaultSize)
chunks = append(chunks, c)
log.Trace("generate random chunk", "idx", i, "chunk", c)
}
@ -460,7 +461,7 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) {
chunks := []*Chunk{}
for i := 0; i < capacity; i++ {
c := GenerateRandomChunk(DefaultChunkSize)
c := GenerateRandomChunk(chunk.DefaultSize)
chunks = append(chunks, c)
log.Trace("generate random chunk", "idx", i, "chunk", c)
}

8
swarm/storage/localstore_test.go
View file

@ -20,6 +20,8 @@ import (
"io/ioutil"
"os"
"testing"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
var (
@ -61,7 +63,7 @@ func TestValidator(t *testing.T) {
// add content address validator and check puts
// bad should fail, good should pass
store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc))
chunks = GenerateRandomChunks(DefaultChunkSize, 2)
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
goodChunk = chunks[0]
badChunk = chunks[1]
copy(badChunk.SData, goodChunk.SData)
@ -79,7 +81,7 @@ func TestValidator(t *testing.T) {
var negV boolTestValidator
store.Validators = append(store.Validators, negV)
chunks = GenerateRandomChunks(DefaultChunkSize, 2)
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
goodChunk = chunks[0]
badChunk = chunks[1]
copy(badChunk.SData, goodChunk.SData)
@ -97,7 +99,7 @@ func TestValidator(t *testing.T) {
var posV boolTestValidator = true
store.Validators = append(store.Validators, posV)
chunks = GenerateRandomChunks(DefaultChunkSize, 2)
chunks = GenerateRandomChunks(chunk.DefaultSize, 2)
goodChunk = chunks[0]
badChunk = chunks[1]
copy(badChunk.SData, goodChunk.SData)

3
swarm/storage/mru/resource_test.go
View file

@ -30,6 +30,7 @@ import (
"github.com/ethereum/go-ethereum/contracts/ens"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/multihash"
"github.com/ethereum/go-ethereum/swarm/storage"
)
@ -783,7 +784,7 @@ func TestValidatorInStore(t *testing.T) {
store.Validators = append(store.Validators, rh)
// create content addressed chunks, one good, one faulty
chunks := storage.GenerateRandomChunks(storage.DefaultChunkSize, 2)
chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2)
goodChunk := chunks[0]
badChunk := chunks[1]
badChunk.SData = goodChunk.SData

5
swarm/storage/pyramid.go
View file

@ -25,6 +25,7 @@ import (
"sync"
"time"
"github.com/ethereum/go-ethereum/swarm/chunk"
"github.com/ethereum/go-ethereum/swarm/log"
)
@ -101,11 +102,11 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get
New chunks to store are store using the putter which the caller provides.
*/
func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, DefaultChunkSize)).Split(ctx)
return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx)
}
func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) {
return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, DefaultChunkSize)).Append(ctx)
return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx)
}
// Entry to create a tree node

5
swarm/storage/types.go
View file

@ -30,6 +30,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/swarm/bmt"
"github.com/ethereum/go-ethereum/swarm/chunk"
)
const MaxPO = 16
@ -230,8 +231,8 @@ func GenerateRandomChunk(dataSize int64) *Chunk {
func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk) {
var i int
hasher := MakeHashFunc(DefaultHash)()
if dataSize > DefaultChunkSize {
dataSize = DefaultChunkSize
if dataSize > chunk.DefaultSize {
dataSize = chunk.DefaultSize
}
for i = 0; i < count; i++ {