swarm/...: Use Bmt hash for swarm (#389)

* swarm/...: Replace SHA3 to BMT

* swarm/storage: Removed temporary logging

* vendor: BMT_SHA3 length is 32

* swarm/storage: Do not ResetWithLength resource hasher

* swarm/storage: Make random chunk creation the same in different tests

And change hash to BMT in hasherstore_test and localstore_test

* swarm/storage: Use BMT hash in common_test

* swarm/storage: Use GenerateRandomChunk in mputChunks too

* swarm/storage, multihash: Package modified multihash with swarm

* vendor: Remove multihash vendored library

* swarm/multihash: Add readme

swarm/multihash/README_SWARM.md

@@ -0,0 +1,5 @@
+# go-multihash library packaged with Swarm
+
+We packaged the go-multihash library with Swarm, because we modified it to add support for BMT
+hash.
+The code is based on the revision ` f1ef5a02f28c862ca5a2037907cf76cc6c98dbf9` of https://github.com/multiformats/go-multihash/

swarm/multihash/multihash.go

@@ -58,6 +58,8 @@ const (
 	BLAKE2S_MIN = 0xb241
 	BLAKE2S_MAX = 0xb260
 
+	BMT_SHA3 = 0xb270
+
 	DBL_SHA2_256 = 0x56
 
 	MURMUR3 = 0x22
@@ -102,6 +104,7 @@ var Names = map[string]uint64{
 	"keccak-512":   KECCAK_512,
 	"shake-128":    SHAKE_128,
 	"shake-256":    SHAKE_256,
+	"bmt-sha3":     BMT_SHA3,
 }
 
 // Codes maps a hash code to it's name
@@ -122,6 +125,7 @@ var Codes = map[uint64]string{
 	KECCAK_512:   "keccak-512",
 	SHAKE_128:    "shake-128",
 	SHAKE_256:    "shake-256",
+	BMT_SHA3:     "bmt-sha3",
 }
 
 // DefaultLengths maps a hash code to it's default length
@@ -142,6 +146,7 @@ var DefaultLengths = map[uint64]int{
 	KECCAK_512:   64,
 	SHAKE_128:    32,
 	SHAKE_256:    64,
+	BMT_SHA3:     32,
 }
 
 func uvarint(buf []byte) (uint64, []byte, error) {

swarm/pss/pss.go

@@ -160,13 +160,13 @@ func NewPss(k network.Overlay, params *PssParams) *Pss {
 		allowRaw: params.AllowRaw,
 		hashPool: sync.Pool{
 			New: func() interface{} {
-				return storage.MakeHashFunc(storage.SHA3Hash)()
+				return storage.MakeHashFunc(storage.DefaultHash)()
 			},
 		},
 	}
 
 	for i := 0; i < hasherCount; i++ {
-		hashfunc := storage.MakeHashFunc(storage.SHA3Hash)()
+		hashfunc := storage.MakeHashFunc(storage.DefaultHash)()
 		ps.hashPool.Put(hashfunc)
 	}
 

swarm/storage/common_test.go

@@ -19,16 +19,13 @@ package storage
 import (
 	"bytes"
 	"crypto/rand"
-	"encoding/binary"
 	"flag"
 	"fmt"
-	"hash"
 	"io"
 	"sync"
 	"testing"
 	"time"
 
-	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/log"
 	colorable "github.com/mattn/go-colorable"
 )
@@ -58,35 +55,10 @@ func brokenLimitReader(data io.Reader, size int, errAt int) *brokenLimitedReader
 	}
 }
 
-func mputChunks(store ChunkStore, processors int, n int, chunksize int, hash hash.Hash) (hs []Key) {
-	f := func(int) *Chunk {
-		data := make([]byte, chunksize)
-		rand.Reader.Read(data)
-		hash.Reset()
-		hash.Write(data)
-		h := hash.Sum(nil)
-		chunk := NewChunk(Key(h), nil)
-		chunk.SData = data
-		return chunk
+func mputChunks(store ChunkStore, processors int, n int, chunksize int64) (hs []Key) {
+	return mput(store, processors, n, GenerateRandomChunk)
 }
-	return mput(store, processors, n, f)
-}
-
-func mputRandomKey(store ChunkStore, processors int, n int, chunksize int) (hs []Key) {
-	data := make([]byte, chunksize+8)
-	binary.LittleEndian.PutUint64(data[0:8], uint64(chunksize))
-
-	f := func(int) *Chunk {
-		h := make([]byte, 32)
-		rand.Reader.Read(h)
-		chunk := NewChunk(Key(h), nil)
-		chunk.SData = data
-		return chunk
-	}
-	return mput(store, processors, n, f)
-}
-
-func mput(store ChunkStore, processors int, n int, f func(i int) *Chunk) (hs []Key) {
+func mput(store ChunkStore, processors int, n int, f func(i int64) *Chunk) (hs []Key) {
 	wg := sync.WaitGroup{}
 	wg.Add(processors)
 	c := make(chan *Chunk)
@@ -108,14 +80,14 @@ func mput(store ChunkStore, processors int, n int, f func(i int) *Chunk) (hs []K
 	}
 	fa := f
 	if _, ok := store.(*MemStore); ok {
-		fa = func(i int) *Chunk {
+		fa = func(i int64) *Chunk {
 			chunk := f(i)
 			chunk.markAsStored()
 			return chunk
 		}
 	}
 	for i := 0; i < n; i++ {
-		chunk := fa(i)
+		chunk := fa(int64(i))
 		hs = append(hs, chunk.Key)
 		c <- chunk
 	}
@@ -180,22 +152,23 @@ func generateRandomData(l int) (r io.Reader, slice []byte) {
 	return
 }
 
-func testStoreRandom(m ChunkStore, processors int, n int, chunksize int, t *testing.T) {
-	hs := mputRandomKey(m, processors, n, chunksize)
+func testStoreRandom(m ChunkStore, processors int, n int, chunksize int64, t *testing.T) {
+	hs := mputChunks(m, processors, n, chunksize)
 	err := mget(m, hs, nil)
 	if err != nil {
 		t.Fatalf("testStore failed: %v", err)
 	}
 }
 
-func testStoreCorrect(m ChunkStore, processors int, n int, chunksize int, t *testing.T) {
-	hs := mputChunks(m, processors, n, chunksize, sha3.NewKeccak256())
+func testStoreCorrect(m ChunkStore, processors int, n int, chunksize int64, t *testing.T) {
+	hs := mputChunks(m, processors, n, chunksize)
 	f := func(h Key, chunk *Chunk) error {
 		if !bytes.Equal(h, chunk.Key) {
 			return fmt.Errorf("key does not match retrieved chunk Key")
 		}
-		hasher := sha3.NewKeccak256()
-		hasher.Write(chunk.SData)
+		hasher := MakeHashFunc(DefaultHash)()
+		hasher.ResetWithLength(chunk.SData[:8])
+		hasher.Write(chunk.SData[8:])
 		exp := hasher.Sum(nil)
 		if !bytes.Equal(h, exp) {
 			return fmt.Errorf("key is not hash of chunk data")
@@ -208,16 +181,16 @@ func testStoreCorrect(m ChunkStore, processors int, n int, chunksize int, t *tes
 	}
 }
 
-func benchmarkStorePut(store ChunkStore, processors int, n int, chunksize int, b *testing.B) {
+func benchmarkStorePut(store ChunkStore, processors int, n int, chunksize int64, b *testing.B) {
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		mputRandomKey(store, processors, n, chunksize)
+		mputChunks(store, processors, n, chunksize)
 	}
 }
 
-func benchmarkStoreGet(store ChunkStore, processors int, n int, chunksize int, b *testing.B) {
-	hs := mputRandomKey(store, processors, n, chunksize)
+func benchmarkStoreGet(store ChunkStore, processors int, n int, chunksize int64, b *testing.B) {
+	hs := mputChunks(store, processors, n, chunksize)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {

swarm/storage/dpa.go

@@ -55,7 +55,7 @@ type DPAParams struct {
 
 func NewDPAParams() *DPAParams {
 	return &DPAParams{
-		Hash: SHA3Hash,
+		Hash: DefaultHash,
 	}
 }
 
@@ -67,7 +67,7 @@ func NewLocalDPA(datadir string, basekey []byte) (*DPA, error) {
 	if err != nil {
 		return nil, err
 	}
-	localStore.Validators = append(localStore.Validators, NewContentAddressValidator(MakeHashFunc(SHA3Hash)))
+	localStore.Validators = append(localStore.Validators, NewContentAddressValidator(MakeHashFunc(DefaultHash)))
 	return NewDPA(localStore, NewDPAParams()), nil
 }
 

swarm/storage/hasherstore_test.go

@@ -18,8 +18,6 @@ package storage
 
 import (
 	"bytes"
-	"crypto/rand"
-	"encoding/binary"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/swarm/storage/encryption"
@@ -44,16 +42,16 @@ func TestHasherStore(t *testing.T) {
 
 	for _, tt := range tests {
 		chunkStore := NewMapChunkStore()
-		hasherStore := NewHasherStore(chunkStore, MakeHashFunc(SHA3Hash), tt.toEncrypt)
+		hasherStore := NewHasherStore(chunkStore, MakeHashFunc(DefaultHash), tt.toEncrypt)
 
 		// Put two random chunks into the hasherStore
-		chunkData1 := generateRandomChunkData(tt.chunkLength)
+		chunkData1 := GenerateRandomChunk(int64(tt.chunkLength)).SData
 		key1, err := hasherStore.Put(chunkData1)
 		if err != nil {
 			t.Fatalf("Expected no error got \"%v\"", err)
 		}
 
-		chunkData2 := generateRandomChunkData(tt.chunkLength)
+		chunkData2 := GenerateRandomChunk(int64(tt.chunkLength)).SData
 		key2, err := hasherStore.Put(chunkData2)
 		if err != nil {
 			t.Fatalf("Expected no error got \"%v\"", err)
@@ -118,11 +116,3 @@ func TestHasherStore(t *testing.T) {
 		}
 	}
 }
-
-func generateRandomChunkData(length int) ChunkData {
-	chunkData := make([]byte, length)
-
-	rand.Read(chunkData)
-	binary.LittleEndian.PutUint64(chunkData[:8], uint64(len(chunkData)-8))
-	return chunkData
-}

swarm/storage/ldbstore_test.go

@@ -74,7 +74,7 @@ func (db *testDbStore) close() {
 	}
 }
 
-func testDbStoreRandom(n int, processors int, chunksize int, mock bool, t *testing.T) {
+func testDbStoreRandom(n int, processors int, chunksize int64, mock bool, t *testing.T) {
 	db, err := newTestDbStore(mock, true)
 	if err != nil {
 		t.Fatalf("init dbStore failed: %v", err)
@@ -83,7 +83,7 @@ func testDbStoreRandom(n int, processors int, chunksize int, mock bool, t *testi
 	testStoreRandom(db, processors, n, chunksize, t)
 }
 
-func testDbStoreCorrect(n int, processors int, chunksize int, mock bool, t *testing.T) {
+func testDbStoreCorrect(n int, processors int, chunksize int64, mock bool, t *testing.T) {
 	db, err := newTestDbStore(mock, false)
 	if err != nil {
 		t.Fatalf("init dbStore failed: %v", err)
@@ -166,11 +166,6 @@ func testIterator(t *testing.T, mock bool) {
 	var poc uint
 	chunkkeys := NewKeyCollection(chunkcount)
 	chunkkeys_results := NewKeyCollection(chunkcount)
-	var chunks []*Chunk
-
-	for i := 0; i < chunkcount; i++ {
-		chunks = append(chunks, NewChunk(nil, nil))
-	}
 
 	db, err := newTestDbStore(mock, false)
 	if err != nil {
@@ -178,7 +173,7 @@ func testIterator(t *testing.T, mock bool) {
 	}
 	defer db.close()
 
-	FakeChunk(DefaultChunkSize, chunkcount, chunks)
+	chunks := GenerateRandomChunks(DefaultChunkSize, chunkcount)
 
 	wg := &sync.WaitGroup{}
 	wg.Add(len(chunks))
@@ -226,7 +221,7 @@ func TestMockIterator(t *testing.T) {
 	testIterator(t, true)
 }
 
-func benchmarkDbStorePut(n int, processors int, chunksize int, mock bool, b *testing.B) {
+func benchmarkDbStorePut(n int, processors int, chunksize int64, mock bool, b *testing.B) {
 	db, err := newTestDbStore(mock, true)
 	if err != nil {
 		b.Fatalf("init dbStore failed: %v", err)
@@ -235,7 +230,7 @@ func benchmarkDbStorePut(n int, processors int, chunksize int, mock bool, b *tes
 	benchmarkStorePut(db, processors, n, chunksize, b)
 }
 
-func benchmarkDbStoreGet(n int, processors int, chunksize int, mock bool, b *testing.B) {
+func benchmarkDbStoreGet(n int, processors int, chunksize int64, mock bool, b *testing.B) {
 	db, err := newTestDbStore(mock, true)
 	if err != nil {
 		b.Fatalf("init dbStore failed: %v", err)

swarm/storage/localstore_test.go

@@ -10,34 +10,9 @@ import (
 )
 
 var (
-	hashfunc = MakeHashFunc("SHA3")
+	hashfunc = MakeHashFunc(DefaultHash)
 )
 
-// convenience generator for unique chunks
-type randomChunkGenerator struct {
-	data   []byte
-	hasher SwarmHash
-}
-
-func newRandomChunkGenerator(stem []byte) *randomChunkGenerator {
-	gen := &randomChunkGenerator{
-		data:   make([]byte, 8),
-		hasher: hashfunc(),
-	}
-	gen.data = append(gen.data, stem...)
-	return gen
-}
-
-func (self *randomChunkGenerator) newChunk() *Chunk {
-	self.hasher.Reset()
-	self.hasher.Write(self.data)
-	chunk := NewChunk(self.hasher.Sum(nil), nil)
-	chunk.SData = make([]byte, len(self.data))
-	copy(chunk.SData, self.data)
-	self.data[0]++
-	return chunk
-}
-
 // put to localstore and wait for stored channel
 // does not check delivery error state
 func putChunks(store *LocalStore, chunks ...*Chunk) {
@@ -59,8 +34,6 @@ func putChunks(store *LocalStore, chunks ...*Chunk) {
 // tests that resource update chunks are passed through content address validator
 // the test checking the resouce update validator internal correctness is found in resource_test.go
 func TestValidator(t *testing.T) {
-	bogusData := []byte("00000000bar")
-
 	// set up localstore
 	datadir, err := ioutil.TempDir("", "storage-testvalidator")
 	if err != nil {
@@ -76,10 +49,10 @@ func TestValidator(t *testing.T) {
 	}
 
 	// check puts with no validators, both succeed
-	gen := newRandomChunkGenerator([]byte("foo"))
-	goodChunk := gen.newChunk()
-	badChunk := gen.newChunk()
-	badChunk.SData = bogusData
+	chunks := GenerateRandomChunks(259, 2)
+	goodChunk := chunks[0]
+	badChunk := chunks[1]
+	copy(badChunk.SData, goodChunk.SData)
 
 	putChunks(store, goodChunk, badChunk)
 	if err := goodChunk.GetErrored(); err != nil {
@@ -92,9 +65,10 @@ func TestValidator(t *testing.T) {
 	// add content address validator and check puts
 	// bad should fail, good should pass
 	store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc))
-	goodChunk = gen.newChunk()
-	badChunk = gen.newChunk()
-	badChunk.SData = bogusData
+	chunks = GenerateRandomChunks(DefaultChunkSize, 2)
+	goodChunk = chunks[0]
+	badChunk = chunks[1]
+	copy(badChunk.SData, goodChunk.SData)
 
 	putChunks(store, goodChunk, badChunk)
 	if err := goodChunk.GetErrored(); err != nil {
@@ -113,7 +87,7 @@ func TestValidator(t *testing.T) {
 	}
 	store.Validators = append(store.Validators, rh)
 
-	goodChunk = gen.newChunk()
+	goodChunk = GenerateRandomChunk(DefaultChunkSize)
 	key := rh.resourceHash(42, 1, ens.EnsNode("xyzzy.eth"))
 	data := []byte("bar")
 	uglyChunk := newUpdateChunk(key, nil, 42, 1, "xyzzy.eth", data, len(data))
@@ -135,7 +109,7 @@ func TestValidator(t *testing.T) {
 	store.Validators[0] = store.Validators[1]
 	store.Validators = store.Validators[:1]
 
-	goodChunk = gen.newChunk()
+	goodChunk = GenerateRandomChunk(DefaultChunkSize)
 	key = rh.resourceHash(42, 2, ens.EnsNode("xyzzy.eth"))
 	data = []byte("baz")
 	uglyChunk = newUpdateChunk(key, nil, 42, 2, "xyzzy.eth", data, len(data))

swarm/storage/memstore_test.go

@@ -32,13 +32,13 @@ func newTestMemStore() *MemStore {
 	return NewMemStore(storeparams, nil)
 }
 
-func testMemStoreRandom(n int, processors int, chunksize int, t *testing.T) {
+func testMemStoreRandom(n int, processors int, chunksize int64, t *testing.T) {
 	m := newTestMemStore()
 	defer m.Close()
 	testStoreRandom(m, processors, n, chunksize, t)
 }
 
-func testMemStoreCorrect(n int, processors int, chunksize int, t *testing.T) {
+func testMemStoreCorrect(n int, processors int, chunksize int64, t *testing.T) {
 	m := newTestMemStore()
 	defer m.Close()
 	testStoreCorrect(m, processors, n, chunksize, t)
@@ -78,13 +78,13 @@ func TestMemStoreNotFound(t *testing.T) {
 	}
 }
 
-func benchmarkMemStorePut(n int, processors int, chunksize int, b *testing.B) {
+func benchmarkMemStorePut(n int, processors int, chunksize int64, b *testing.B) {
 	m := newTestMemStore()
 	defer m.Close()
 	benchmarkStorePut(m, processors, n, chunksize, b)
 }
 
-func benchmarkMemStoreGet(n int, processors int, chunksize int, b *testing.B) {
+func benchmarkMemStoreGet(n int, processors int, chunksize int64, b *testing.B) {
 	m := newTestMemStore()
 	defer m.Close()
 	benchmarkStoreGet(m, processors, n, chunksize, b)

swarm/storage/resource_test.go

@@ -14,8 +14,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/multiformats/go-multihash"
-
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
 	"github.com/ethereum/go-ethereum/common"
@@ -25,6 +23,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/multihash"
 )
 
 var (

swarm/storage/swarmhasher.go

@@ -23,6 +23,7 @@ import (
 const (
 	BMTHash     = "BMT"
 	SHA3Hash    = "SHA3" // http://golang.org/pkg/hash/#Hash
+	DefaultHash = BMTHash
 )
 
 type SwarmHash interface {

swarm/storage/types.go

@@ -222,24 +222,29 @@ func (c *Chunk) WaitToStore() {
 	<-c.dbStoredC
 }
 
-func FakeChunk(size int64, count int, chunks []*Chunk) int {
+func GenerateRandomChunk(dataSize int64) *Chunk {
+	return GenerateRandomChunks(dataSize, 1)[0]
+}
+
+func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk) {
 	var i int
-	hasher := MakeHashFunc(SHA3Hash)()
-	if size > DefaultChunkSize {
-		size = DefaultChunkSize
+	hasher := MakeHashFunc(DefaultHash)()
+	if dataSize > DefaultChunkSize {
+		dataSize = DefaultChunkSize
 	}
 
 	for i = 0; i < count; i++ {
-		hasher.Reset()
-		chunks[i].SData = make([]byte, size)
+		chunks = append(chunks, NewChunk(nil, nil))
+		chunks[i].SData = make([]byte, dataSize+8)
 		rand.Read(chunks[i].SData)
-		binary.LittleEndian.PutUint64(chunks[i].SData[:8], uint64(size))
-		hasher.Write(chunks[i].SData)
+		binary.LittleEndian.PutUint64(chunks[i].SData[:8], uint64(dataSize))
+		hasher.ResetWithLength(chunks[i].SData[:8])
+		hasher.Write(chunks[i].SData[8:])
 		chunks[i].Key = make([]byte, 32)
 		copy(chunks[i].Key, hasher.Sum(nil))
 	}
 
-	return i
+	return chunks
 }
 
 // Size, Seek, Read, ReadAt
@@ -275,7 +280,7 @@ func NewStoreParams(ldbCap uint64, cacheCap uint, requestsCap uint, hash SwarmHa
 		basekey = make([]byte, 32)
 	}
 	if hash == nil {
-		hash = MakeHashFunc("SHA3")
+		hash = MakeHashFunc(DefaultHash)
 	}
 	return &StoreParams{
 		Hash:                       hash,

swarm/swarm.go

@@ -192,7 +192,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
 	}
 
 	var validators []storage.ChunkValidator
-	validators = append(validators, storage.NewContentAddressValidator(storage.MakeHashFunc(storage.SHA3Hash)))
+	validators = append(validators, storage.NewContentAddressValidator(storage.MakeHashFunc(storage.DefaultHash)))
 	if resourceHandler != nil {
 		validators = append(validators, resourceHandler)
 	}

swarm/testutil/http.go

@@ -78,7 +78,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
 		Server: srv,
 		Dpa:    dpa,
 		dir:    dir,
-		Hasher: storage.MakeHashFunc(storage.SHA3Hash)(),
+		Hasher: storage.MakeHashFunc(storage.DefaultHash)(),
 		cleanup: func() {
 			srv.Close()
 			rh.Close()

vendor/vendor.json

@@ -285,12 +285,6 @@
 			"revision": "ad45545899c7b13c020ea92b2072220eefad42b8",
 			"revisionTime": "2015-03-14T17:03:34Z"
 		},
-		{
-			"checksumSHA1": "qcDnp9aAjAswDFGztkzgb+BZA+0=",
-			"path": "github.com/multiformats/go-multihash",
-			"revision": "f1ef5a02f28c862ca5a2037907cf76cc6c98dbf9",
-			"revisionTime": "2017-07-13T18:40:44Z"
-		},
 		{
 			"checksumSHA1": "FYM/8R2CqS6PSNAoKl6X5gNJ20A=",
 			"path": "github.com/naoina/toml",