forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-20 02:42:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

cmd/swarm/swarm-smoke: address PR comments

Browse source
This commit is contained in:
Elad Nachmias 2019-01-28 13:20:47 +07:00
parent ca37153f3a
commit 75b2b3428a
5 changed files with 59 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
cmd/swarm/swarm-smoke/feed_upload_and_sync.go
View file

@ -3,6 +3,7 @@ package main
import (
"bytes"
"crypto/md5"
crand "crypto/rand"
"fmt"
"io"
"io/ioutil"
@ -16,7 +17,6 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
"github.com/ethereum/go-ethereum/swarm/testutil"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
@ -203,9 +203,10 @@ func feedUploadAndSync(c *cli.Context) error {
seed := int(time.Now().UnixNano() / 1e6)
log.Info("feed uploading to "+endpoints[0]+" and syncing", "seed", seed)
randomBytes := testutil.RandomBytes(seed, filesize*1000)
h = md5.New()
r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h)
hash, err := upload(&randomBytes, endpoints[0])
hash, err := upload(r, filesize*1000, endpoints[0])
if err != nil {
return err
}
@ -214,10 +215,7 @@ func feedUploadAndSync(c *cli.Context) error {
return err
}
multihashHex := hexutil.Encode(hashBytes)
fileHash, err := digest(bytes.NewReader(randomBytes))
if err != nil {
return err
}
fileHash := h.Sum(nil)
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash))

63
cmd/swarm/swarm-smoke/sliding_window.go
View file

@ -17,31 +17,32 @@
package main
import (
"bytes"
"crypto/md5"
crand "crypto/rand"
"fmt"
"io"
"math/rand"
"sync"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/testutil"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
var seed = time.Now().UTC().UnixNano()
func init() {
rand.Seed(seed)
}
type uploadResult struct {
hash string
digest []byte
}
func slidingWindow(c *cli.Context) error {
// test dscription:
// 1. upload repeatedly the same file size, maintain a slice in which swarm hashes are stored, first hash at idx=0
// 2. select a random node, start downloading the hashes, starting with the LAST one first (it should always be availble), till the FIRST hash
// 3. when
defer func(now time.Time) {
totalTime := time.Since(now)
@ -50,69 +51,61 @@ func slidingWindow(c *cli.Context) error {
}(time.Now())
generateEndpoints(scheme, cluster, appName, from, to)
storeSize = storeSize * 4096 //store size is in chunks - transform to bytes
hashes := []uploadResult{} //swarm hashes of the uploads
nodes := to - from
networkCapacity := float64(storeSize) * float64(nodes)
const iterationTimeout = 30 * time.Second
log.Info("sliding window test started", "store size(kb)", int(storeSize/1000), "nodes", nodes, "filesize(kb)", filesize, "network capacity(kb)", int(networkCapacity/1000), "timeout", timeout)
log.Info("sliding window test started", "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout)
uploadedBytes := 0
networkDepth := 0
errored := false
outer:
for {
seed := int(time.Now().UnixNano() / 1e6)
log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed)
randomBytes := testutil.RandomBytes(seed, filesize*1000)
h := md5.New()
r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h)
t1 := time.Now()
hash, err := upload(&randomBytes, endpoints[0])
if err != nil {
log.Error(err.Error())
return err
}
metrics.GetOrRegisterCounter("sliding-window.upload-time", nil).Inc(int64(time.Since(t1)))
fhash, err := digest(bytes.NewReader(randomBytes))
hash, err := upload(r, filesize*1000, endpoints[0])
if err != nil {
log.Error(err.Error())
return err
}
metrics.GetOrRegisterResettingTimer("sliding-window.upload-time", nil).UpdateSince(t1)
fhash := h.Sum(nil)
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash), "sleeping", syncDelay)
hashes = append(hashes, uploadResult{hash: hash, digest: fhash})
time.Sleep(time.Duration(syncDelay) * time.Second)
uploadedBytes += filesize * 1000
var wg sync.WaitGroup
for i, v := range hashes {
timeout := time.After(30 * time.Second)
errored = false
wg.Add(1)
go func(i int, v uploadResult) {
defer wg.Done()
inner:
for {
select {
case <-timeout:
errored = true
log.Error("error retrieving hash. timeout", "hash idx", i, "err", err)
metrics.GetOrRegisterCounter("sliding-window.single.error", nil).Inc(1)
return
default:
}
rand.Seed(time.Now().UTC().UnixNano())
randIndex := 1 + rand.Intn(len(endpoints)-1)
ruid := uuid.New()[:8]
start := time.Now()
err := fetch(v.hash, endpoints[randIndex], v.digest, ruid)
fetchTime := time.Since(start)
if err != nil {
continue
continue inner
}
metrics.GetOrRegisterMeter("sliding-window.single.fetch-time", nil).Mark(int64(fetchTime))
metrics.GetOrRegisterResettingTimer("sliding-window.single.fetch-time", nil).UpdateSince(start)
break inner
}
}(i, v)
wg.Wait()
}
if errored {
break outer
}
@ -120,8 +113,8 @@ outer:
}
}
log.Info("sliding window test finished", "errored?", errored, "networkDepth", networkDepth, "networkDepth(kb)", int(networkDepth*filesize))
log.Info("stats", "uploadedFiles", len(hashes), "uploadedKb", uploadedBytes/1000, "filesizeKb", filesize, "networkCapacityKb", int(networkCapacity/1000), "networkCapacityMb", int(networkCapacity/1000000))
log.Info("sliding window test finished", "errored?", errored, "networkDepth", networkDepth, "networkDepth(kb)", networkDepth*filesize)
log.Info("stats", "uploadedFiles", len(hashes), "uploadedKb", uploadedBytes/1000, "filesizeKb", filesize)
metrics.GetOrRegisterMeter("sliding-window.network-depth", nil).Mark(int64(networkDepth))
return nil

17
cmd/swarm/swarm-smoke/upload_and_sync.go
View file

@ -17,15 +17,16 @@
package main
import (
"bytes"
"crypto/md5"
crand "crypto/rand"
"fmt"
"io"
"math/rand"
"sync"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/testutil"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
@ -40,23 +41,21 @@ func uploadAndSync(c *cli.Context) error {
generateEndpoints(scheme, cluster, appName, from, to)
seed := int(time.Now().UnixNano() / 1e6)
log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed)
randomBytes := testutil.RandomBytes(seed, filesize*1000)
h := md5.New()
r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h)
t1 := time.Now()
hash, err := upload(&randomBytes, endpoints[0])
hash, err := upload(r, filesize*1000, endpoints[0])
if err != nil {
log.Error(err.Error())
return err
}
metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).UpdateSince(t1)
fhash, err := digest(bytes.NewReader(randomBytes))
if err != nil {
log.Error(err.Error())
return err
}
fhash := h.Sum(nil)
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash))

16
cmd/swarm/swarm-smoke/upload_speed.go
View file

@ -17,13 +17,14 @@
package main
import (
"bytes"
"crypto/md5"
crand "crypto/rand"
"fmt"
"io"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/testutil"
cli "gopkg.in/urfave/cli.v1"
)
@ -40,21 +41,18 @@ func uploadSpeed(c *cli.Context) error {
seed := int(time.Now().UnixNano() / 1e6)
log.Info("uploading to "+endpoint, "seed", seed)
randomBytes := testutil.RandomBytes(seed, filesize*1000)
h := md5.New()
r := io.TeeReader(io.LimitReader(crand.Reader, int64(filesize*1000)), h)
t1 := time.Now()
hash, err := upload(&randomBytes, endpoint)
hash, err := upload(r, filesize*1000, endpoint)
if err != nil {
log.Error(err.Error())
return err
}
metrics.GetOrRegisterCounter("upload-speed.upload-time", nil).Inc(int64(time.Since(t1)))
fhash, err := digest(bytes.NewReader(randomBytes))
if err != nil {
log.Error(err.Error())
return err
}
fhash := h.Sum(nil)
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash))
return nil

6
cmd/swarm/swarm-smoke/util.go
View file

@ -203,14 +203,14 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error {
}
// upload an arbitrary byte as a plaintext file to `endpoint` using the api client
func upload(dataBytes *[]byte, endpoint string) (string, error) {
func upload(r io.Reader, size int, endpoint string) (string, error) {
swarm := client.NewClient(endpoint)
f := &client.File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(*dataBytes)),
ReadCloser: ioutil.NopCloser(r),
ManifestEntry: api.ManifestEntry{
ContentType: "text/plain",
Mode: 0660,
Size: int64(len(*dataBytes)),
Size: int64(size),
},
}