From 185122326826c4cd29e18a3f74c9081855a18eaa Mon Sep 17 00:00:00 2001 From: Oren Date: Fri, 7 Apr 2017 12:52:06 +0300 Subject: [PATCH] swarm/storage - Inclusion proof for any given offset and length within a specific chunk -Performance improvements --- swarm/storage/binarymerkle.go | 410 +++++++++++++++++------------ swarm/storage/binarymerkle_test.go | 214 +++++++++++---- 2 files changed, 393 insertions(+), 231 deletions(-) diff --git a/swarm/storage/binarymerkle.go b/swarm/storage/binarymerkle.go index c33fe3d1a9..9daca3e008 100644 --- a/swarm/storage/binarymerkle.go +++ b/swarm/storage/binarymerkle.go @@ -5,28 +5,25 @@ package storage import ( "bytes" _ "crypto/sha256" - "encoding/binary" "errors" "fmt" "hash" "io" - "sync" "github.com/ethereum/go-ethereum/crypto/sha3" ) var hashFunc Hasher = sha3.NewKeccak256 //default hasher +const ( + segmentsize int = 32 +) + type state struct { btree BTree root Root } -const ( - // max number of leafs on 4096 chunk len in 32 bytes segments - mprocessors = 128 -) - // A merkle tree for a user that stores the entire tree // Specifically this tree is left a leaning balanced binary tree // Where each node holds the hash of its leaves @@ -36,6 +33,7 @@ type BTree struct { count uint64 root *node rootHash []byte + chunklen int //hashFunc Hasher } @@ -52,17 +50,15 @@ type node struct { } type jobparam struct { - data [][]byte - height int - n0 *node - n1 *node - id int + //data [][]byte + n0 *node + n1 *node + id int } type jobresult struct { - n *node - data [][]byte - id int + n *node + id int } func (t BTree) Count() uint64 { @@ -141,20 +137,21 @@ func rootHash(count uint64, data []byte) []byte { h := hashFunc() h.Reset() h.Write(data) - binary.Write(h, binary.LittleEndian, count) + //binary.Write(h, binary.LittleEndian, count) return h.Sum(nil) } func makeHash(left, right *node) []byte { h := hashFunc() h.Reset() + if left != nil { h.Write(left.label) if right != nil { h.Write(right.label) } } - return h.Sum(make([]byte, 0)) + return h.Sum(nil) } // Returns the height of the tree containing count leaf nodes. @@ -176,161 +173,173 @@ func GetHeight(count uint64) int { // BMT - The BMT Representation of the data // ROOT - BMT Root // Count - Numers of leafs at the BMT -// error - -func BuildBMT(h Hasher, data []byte, segmentsize int) (bmt *BTree, roor *Root, count int, err error) { - if len(data) == 0 { +// err +// The bmt computation is done in parallel by deviding the tree to subtree . +// (each sub tree is calculated in parallel using goroutine ) and then merge the results in parallel and get the tree. +// The paralel merging is done by creating seperate channel for each node and make it to wait(on a seperate go routine) for the calculation of +// its left and right childerens. +func BuildBMT(h Hasher, data []byte, validate bool) (bmt *BTree, roor *Root, count int, err error) { + + if (len(data) & (len(data) - 1)) == 0 { //check if power of 2 + return buildBMTfaster(h, data, validate) + } else { + return buildBMTfast(h, data, validate) + } +} + +//This function assume its data len value is a power of 2 +func buildBMTfaster(h Hasher, data []byte, validate bool) (bmt *BTree, roor *Root, count int, err error) { + + datalen := len(data) + if datalen == 0 { return nil, nil, 0, errors.New("data length is 0 ") } hashFunc = h - leafcount := len(data) / segmentsize - if len(data)%segmentsize != 0 { + leafcount := datalen / segmentsize + if datalen%segmentsize != 0 { leafcount++ } - tree := BuildParralel(data, segmentsize) - err = tree.Validate() - if err != nil { - return nil, nil, 0, errors.New("Validation error") - } - if tree.Count() != uint64(leafcount) { - return nil, nil, 0, errors.New("Validation count error") + var rootnode *node + var subtreescount = 4 + //setting the subtreescount to 4 yield the best benchmarks results. + if leafcount < 4 { + subtreescount = 2 } + if leafcount > 1 { + subtreesize := datalen / subtreescount + subtreeleafcount := subtreesize / segmentsize + if subtreesize%segmentsize != 0 { + subtreeleafcount++ + } + subtreeheight := GetHeight(uint64(subtreeleafcount)) + height := GetHeight(uint64(subtreescount)) + results := make([]chan *node, (1 << uint(height))) //array of channels for each node + //start := time.Now() + + for i := 0; i < subtreescount; i++ { + results[i] = make(chan *node) + results[subtreescount+i] = make(chan *node) + go func(subdata []byte, index int) { + subtreerootnode, _ := buildNode(subdata, subtreeheight) + results[index] <- subtreerootnode + }(data[i*subtreesize:(i+1)*subtreesize], i) + } + for i := 0; i < subtreescount-1; i++ { + go func(index int, resultindex int) { + var leftnode, rightnode *node + select { + case leftnode = <-results[index]: + rightnode = <-results[index+1] + case rightnode = <-results[index+1]: + leftnode = <-results[index] + } + results[resultindex] <- &node{label: makeHash(leftnode, rightnode), + children: [2]*node{leftnode, rightnode}} + }(i*2, subtreescount+i) + } + + rootnode = <-results[(1< 1 { + subtreesize := leafcount / subtreescount + subtreeheight := GetHeight(uint64(subtreesize)) + height := GetHeight(uint64(subtreescount)) + results := make([]chan *node, (1 << uint(height))) //array of channels for each node + + for i := 0; i < subtreescount; i++ { + results[i] = make(chan *node) + results[subtreescount+i] = make(chan *node) + go func(subdata [][]byte, index int) { + subtreerootnode, _ := buildNode2(subdata, subtreeheight) + results[index] <- subtreerootnode + }(blocks[i*(leafcount/subtreescount):(i+1)*(leafcount/subtreescount)], i) + } + + for i := 0; i < subtreescount-1; i++ { + go func(index int, resultindex int) { + var leftnode, rightnode *node + select { + case leftnode = <-results[index]: + rightnode = <-results[index+1] + case rightnode = <-results[index+1]: + leftnode = <-results[index] + } + results[resultindex] <- &node{label: makeHash(leftnode, rightnode), + children: [2]*node{leftnode, rightnode}} + }(i*2, subtreescount+i) + + } + + rootnode = <-results[(1< 0 { - rootLabel = node.label - } - hash := rootHash(count, rootLabel) - t := BTree{count, node, hash} - return &t -} - -// Build a tree parralel computation -// We build it level by level ..starting from the depper one and get up.. -func BuildParralel(data []byte, segmentsize int) *BTree { - count := len(data) / segmentsize - var height int - - if len(data)%segmentsize != 0 { - height = GetHeight(uint64(count + 1)) - - } else { - height = GetHeight(uint64(count)) - } - - jobs := make(chan jobparam, mprocessors) - results := make(chan jobresult, mprocessors) - - pend := new(sync.WaitGroup) - - for w := 1; w <= mprocessors; w++ { - pend.Add(1) - go processor(pend, 2, jobs, results) - } - /*build leafs*/ - var leaflevelcount = 1 << uint(height-1) - for k := 0; k < leaflevelcount; k++ { - datasegment := make([][]byte, 0, 1) - pend.Add(1) - if k*segmentsize+segmentsize > len(data) { - if (k*segmentsize + len(data)%segmentsize) > len(data) { - jobs <- jobparam{data: nil, height: 0, n0: nil, n1: nil, id: k} - } else { - datasegment = append(datasegment, data[k*segmentsize:k*segmentsize+len(data)%segmentsize]) - jobs <- jobparam{data: datasegment, height: 1, n0: nil, n1: nil, id: k} - } - } else { - datasegment = append(datasegment, data[k*segmentsize:(k+1)*segmentsize]) - jobs <- jobparam{data: datasegment, height: 1, n0: nil, n1: nil, id: k} - } - } - fmt.Println("1") - close(jobs) - pend.Wait() - - if len(data)%segmentsize != 0 { - count++ - } - restmp := make([]jobresult, leaflevelcount) - fmt.Println("2", leaflevelcount) - //Get computation results - for i := 0; i < leaflevelcount; i++ { - tres := <-results - restmp[tres.id] = tres - } - fmt.Println("3", height) - for i := 1; i < height; i++ { - jobs := make(chan jobparam, mprocessors) - for w := 1; w <= mprocessors; w++ { - pend.Add(1) - go processor(pend, 2, jobs, results) - } - var indexres = 0 - for j := 0; j < (1<>uint(i); j++ { - pend.Add(1) - jobs <- jobparam{n0: restmp[indexres].n, n1: restmp[indexres+1].n, data: restmp[indexres+1].data, height: 2, id: j} - indexres += 2 - } - close(jobs) - pend.Wait() - if i < height-1 { - for l := 0; l < (1<>uint(i); l++ { - tmpres := <-results - restmp[tmpres.id] = tmpres - } - } - } - rootLabel := make([]byte, 0) - var res jobresult - if leaflevelcount == 1 { - res = restmp[0] - } else { - res = <-results - } - if height > 0 { - rootLabel = res.n.label - } - hash := rootHash(uint64(count), rootLabel) - t := BTree{uint64(count), res.n, hash} - return &t -} - // returns a node and the left over data not used by it -func buildNode(data [][]byte, height int) (*node, [][]byte) { +func buildNode(data []byte, height int) (*node, []byte) { + if height == 0 || len(data) == 0 { + return nil, data + } + if height == 1 { + // leaf + return &node{label: data[0:segmentsize]}, data[segmentsize:] + } + n0, data := buildNode(data, height-1) + n1, data := buildNode(data, height-1) + + hash := makeHash(n0, n1) + return &node{label: hash, children: [2]*node{n0, n1}}, data +} + +func buildNode2(data [][]byte, height int) (*node, [][]byte) { if height == 0 || len(data) == 0 { return nil, data } @@ -338,8 +347,8 @@ func buildNode(data [][]byte, height int) (*node, [][]byte) { // leaf return &node{label: data[0]}, data[1:] } - n0, data := buildNode(data, height-1) - n1, data := buildNode(data, height-1) + n0, data := buildNode2(data, height-1) + n1, data := buildNode2(data, height-1) hash := makeHash(n0, n1) return &node{label: hash, children: [2]*node{n0, n1}}, data @@ -356,38 +365,80 @@ func splitData(data []byte, size int) [][]byte { if len(data)%size != 0 { blocks = append(blocks, data[len(blocks)*size:]) } + height := GetHeight(uint64(len(blocks))) + for i := len(blocks); i < (1 << uint(height)); i++ { + blocks = append(blocks, nil) + } + // return blocks } +type inclusionproofs struct { + proofs []inclusionproof + offset int + len int +} + +type inclusionproof struct { + proof [][]byte + offset int + len int + index int +} + +func (t *BTree) GetInclusionProofs(offset int, length int) (proofs inclusionproofs, err error) { + + if offset+length > t.chunklen { + + return proofs, errors.New(fmt.Sprintf("wrong offset+len %d :chunklen:%d", offset+length, t.chunklen)) + } + + n := (offset%segmentsize+length)/segmentsize + 1 + + proofs.proofs = make([]inclusionproof, n+1) + var index int = 0 + var segment = offset / segmentsize + for i := segment; i <= segment+n; i++ { + proofs.proofs[index], err = t.InclusionProof(i) + index++ + } + proofs.len = length + proofs.offset = offset + return proofs, nil +} + // Return a [][]byte needed to prove the gkf of the item at the passed index // The payload of the item at index is the first value in the proof -func (t *BTree) InclusionProof(index int) [][]byte { +func (t *BTree) InclusionProof(index int) (proof inclusionproof, err error) { if uint64(index) >= t.count { - panic("Invalid index: too large") + return proof, errors.New("Invalid index: too large") } if index < 0 { - panic("Invalid index: negative") + return proof, errors.New("Invalid index: negative") } h := GetHeight(t.count) - fmt.Println(h) - return proveNode(h, t.root, index) + proof.proof, err = proveNode(h, t.root, index) + proof.offset = index * segmentsize + proof.len = segmentsize + proof.index = index + return proof, err } -func proveNode(height int, n *node, index int) [][]byte { +func proveNode(height int, n *node, index int) ([][]byte, error) { if height == 1 { if index != 0 { - panic("Invalid index: non 0 for final node") + return nil, errors.New("Invalid index: non 0 for final node") } - return [][]byte{n.label} + return [][]byte{n.label}, nil } childIndex := index >> uint(height-2) nextIndex := index & (^(1 << uint(height-2))) - b := proveNode(height-1, n.children[childIndex], nextIndex) + b, _ := proveNode(height-1, n.children[childIndex], nextIndex) otherChildIndex := (childIndex + 1) % 2 if n.children[otherChildIndex] != nil { b = append(b, n.children[otherChildIndex].label) } - return b + return b, nil } // The Root of a merkle tree for a client that does not store the tree @@ -396,6 +447,18 @@ type Root struct { Base []byte } +func (r *Root) CheckProofs(h Hasher, proofs inclusionproofs) (bool, error) { + n := (proofs.offset%segmentsize+proofs.len)/segmentsize + 1 + + for i := 0; i < n; i++ { + ok, err := r.CheckProof(h, proofs.proofs[i].proof, proofs.proofs[i].index) + if (ok == false) || (err != nil) { + return ok, err + } + } + return true, nil +} + // Proves theof an element at the given index with the value thats the first entry in proof func (r *Root) CheckProof(h Hasher, proof [][]byte, index int) (bool, error) { hashFunc = h @@ -484,7 +547,8 @@ func (d *state) Reset() { // Write absorbs more data into the hash's state. func (d *state) Write(p []byte) (written int, err error) { - tree, r, count, err1 := BuildBMT(hashFunc, p, 32) + + tree, r, count, err1 := BuildBMT(hashFunc, p, true) d.btree = *tree d.root = *r diff --git a/swarm/storage/binarymerkle_test.go b/swarm/storage/binarymerkle_test.go index ec604db583..d78f0b0b02 100644 --- a/swarm/storage/binarymerkle_test.go +++ b/swarm/storage/binarymerkle_test.go @@ -3,84 +3,182 @@ package storage import ( "fmt" "log" + "math/rand" "testing" "time" "github.com/ethereum/go-ethereum/crypto/sha3" ) -//will test the hash (hash.Hash) interface -func TestBuildBMT2(t *testing.T) { +func TestBuildBMT(t *testing.T) { + for n := 0; n <= 4096; n += 1 { + fmt.Println("chunksize", n) + testBuildBMTprv(n, t) + } +} - // Grab some data to make the tree out of, and partition - data := make([]byte, 4096) - tdata := testDataReader(4096) +func testBuildBMTprv(n int, t *testing.T) { + + data := make([]byte, n) + tdata := testDataReader(n) tdata.Read(data) - var thashFunc = MakeHashFunc("BMTSHA3") - var h = thashFunc() - h.Reset() - h.Write(data) - var key = h.Sum(nil) - fmt.Println(key) + var tree *BTree + var r *Root + var count int + var err1 error + start := time.Now() + tree, r, count, err1 = BuildBMT(sha3.NewKeccak256, data, true) + elapsed := time.Since(start) + log.Printf("n=%d took %s", n, elapsed) + if err1 != nil { + fmt.Println(tree, r, count, err1) + return + } // for i := 0; i < count; i++ { - // p := tree.InclusionProof(i) + // p, err := tree.InclusionProof(i) + // if err != nil { + // fmt.Println("proof failed ", i, err.Error()) + // continue + // } + // ok, err := r.CheckProof(sha3.NewKeccak256, p.proof, i) // - // fmt.Println(p) - // - // ok := r.CheckProof(sha3.NewKeccak256, p, i) - // if !ok { + // if !ok || (err != nil) { // t.Errorf("proof %d failed", i) // } // } - // fmt.Println("done") -} -func TestBuildBMTStress(t *testing.T) { - - for i := 1; i < 4096; i++ { - BuildBMTwithGivenDataLen(i, t) - } -} - -func TestBuildBMT(t *testing.T) { - BuildBMTwithGivenDataLen(4096, t) -} - -func BuildBMTwithGivenDataLen(datalen int, t *testing.T) { - - // Grab some data to make the tree out of, and partition - fmt.Println("dl", datalen) - data := make([]byte, datalen) - tdata := testDataReader(datalen) - tdata.Read(data) - fmt.Println(data) - - start := time.Now() - - tree, r, count, err1 := BuildBMT(sha3.NewKeccak256, data, 32) - - elapsed := time.Since(start) - log.Printf("Binomial took %s", elapsed) - - if err1 != nil { - fmt.Println(err1) + offset := rand.Intn(n) + length := rand.Intn((n-offset+1)-1) + 1 + p, err := tree.GetInclusionProofs(offset, length) + if err != nil { + t.Errorf("proof %d failed %s", offset, err) return + } - fmt.Println(tree.Root(), count) + ok, err := r.CheckProofs(sha3.NewKeccak256, p) - for i := 0; i < count; i++ { - p := tree.InclusionProof(i) - - fmt.Println(p) - - ok, err := r.CheckProof(sha3.NewKeccak256, p, i) - - if !ok || (err != nil) { - t.Errorf("proof %d failed", i) - } + if !ok || (err != nil) { + t.Errorf("proof failed %s", err) + } else { + fmt.Println("proofs ok for offset", offset, "lenght", length, "chunksize", n) } + + // ok, err := r.CheckProof(sha3.NewKeccak256, p.proof, i) + // + // if !ok || (err != nil) { + // t.Errorf("proof %d failed", i) + // } + fmt.Println("done") } + +func benchmarkBuildBMT(n int, t *testing.B) { + //t.ReportAllocs() + tdata := testDataReader(n) + data := make([]byte, n) + tdata.Read(data) + + //reader := bytes.NewReader(data) + + var tree *BTree + var r *Root + var count int + var err1 error + // blocks := splitData(data, 32) + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + + tree, r, count, err1 = BuildBMT(sha3.NewKeccak256, data, false) + + if err1 != nil { + fmt.Println(err1, tree, r, count) + return + } + } +} + +func benchmarkSHA3(n int, t *testing.B) { + + data := make([]byte, n) + tdata := testDataReader(n) + tdata.Read(data) + hashFunc = sha3.NewKeccak256 + + t.ReportAllocs() + t.ResetTimer() + + h := hashFunc() + for i := 0; i < t.N; i++ { + + h.Reset() + h.Write(data) + //binary.Write(h, binary.LittleEndian, count) + h.Sum(nil) + + } + +} + +func BenchmarkBuildBMT_4k(t *testing.B) { benchmarkBuildBMT(4096, t) } +func BenchmarkBuildBMT_2k(t *testing.B) { benchmarkBuildBMT(4096/2, t) } +func BenchmarkBuildBMT_1k(t *testing.B) { benchmarkBuildBMT(4096/4, t) } +func BenchmarkBuildBMT_512b(t *testing.B) { benchmarkBuildBMT(4096/8, t) } +func BenchmarkBuildBMT_256b(t *testing.B) { benchmarkBuildBMT(4096/16, t) } +func BenchmarkBuildBMT_128b(t *testing.B) { benchmarkBuildBMT(4096/64, t) } + +func BenchmarkBuildSHA3_4k(t *testing.B) { benchmarkSHA3(4096, t) } +func BenchmarkBuildSHA3_2k(t *testing.B) { benchmarkSHA3(4096/2, t) } +func BenchmarkBuildSHA3_1k(t *testing.B) { benchmarkSHA3(4096/4, t) } +func BenchmarkBuildSHA3_512b(t *testing.B) { benchmarkSHA3(4096/8, t) } +func BenchmarkBuildSHA3_256b(t *testing.B) { benchmarkSHA3(4096/16, t) } + +func BenchmarkBuildNagiBinaryMerkle_4k(t *testing.B) { + n := 4096 + data := make([]byte, n) + tdata := testDataReader(n) + tdata.Read(data) + hashFunc = sha3.NewKeccak256 + + t.ReportAllocs() + t.ResetTimer() + + //h := hashFunc() + for i := 0; i < t.N; i++ { + + BinaryMerkle(data, sha3.NewKeccak256) + + } + +} + +//func BenchmarkBinaryMerkleTree(t *testing.B) { benchmarkBMT(4096, t) } + +// This implementation does not take advantage of any paralellisms and uses +// far more memory than necessary, but it is easy to see that it is correct. +// It can be used for generating test cases for optimized implementations. + +func BinaryMerkle(chunk []byte, hasher Hasher) []byte { + hash := hasher() + section := 2 * hash.Size() + l := len(chunk) + if l > section { + n := l / section + r := l - n*section + hash.Write(chunk[0:r]) + next := hash.Sum(nil) + for r < l { + hash.Reset() + hash.Write(chunk[r : r+section]) + next = hash.Sum(next) + r += section + } + return BinaryMerkle(next, hasher) + } else { + hash.Write(chunk) + return hash.Sum(nil) + } +}