From eab1384b08e73824108216c9ac9532f2895b6bc4 Mon Sep 17 00:00:00 2001 From: Oren Date: Wed, 22 Mar 2017 12:18:10 +0200 Subject: [PATCH 1/7] swarm/storage - binary merle tree with inclusion proof of a give leafs size (32 bytes) --- swarm/storage/binarymerkle.go | 315 +++++++++++++++++++++++++++++ swarm/storage/binarymerkle_test.go | 123 +++++++++++ 2 files changed, 438 insertions(+) create mode 100644 swarm/storage/binarymerkle.go create mode 100644 swarm/storage/binarymerkle_test.go diff --git a/swarm/storage/binarymerkle.go b/swarm/storage/binarymerkle.go new file mode 100644 index 0000000000..0c31db1698 --- /dev/null +++ b/swarm/storage/binarymerkle.go @@ -0,0 +1,315 @@ +package storage + +// provides a binary merkle tree implementation. + +import ( + "bytes" + _ "crypto/sha256" + "encoding/binary" + "fmt" + + "github.com/ethereum/go-ethereum/crypto/sha3" +) + +var hashFunc Hasher = sha3.NewKeccak256 //default hasher + +// A merkle tree for a user that stores the entire tree +// Specifically this tree is left a leaning balanced binary tree +// Where each node holds the hash of its leaves +// And the rootHash is the root node hashed with the count +// This tree is immutable +type BTree struct { + count uint64 + root *node + rootHash []byte + //hashFunc Hasher +} + +type node struct { + label []byte + children [2]*node // if all nil, leaf node + + // Representation invariants: + // if children[0] is nil, children[1] is nil + // if both children non nil: + // label is hash of (children[0].label + children[1].label) + // if leaf: label is arbitrary data + // else if children[1] is nil, label=hash(children[0].label) +} + +func (t BTree) Count() uint64 { + return t.count +} + +// The hash/root of an empty BTree does not matter +func (t BTree) Root() []byte { + return t.rootHash +} + +// All trees should pass , unless they are invalid, which should only happen +// if incorrectly built or modified. +// Checks the rep invariants +func (t BTree) Validate() error { + count, height, error := t.root.validate() + if error != nil { + return error + } + if count != t.count { + return fmt.Errorf("Incorrect count. Was %d, should be %d", t.count, count) + } + if height != GetHeight(count) { + return fmt.Errorf("Incorrect height. Was %d, should be %d", height, GetHeight(count)) + } + + rootLabel := make([]byte, 0) + if height > 0 { + rootLabel = t.root.label + } + h := rootHash(count, rootLabel) + if !bytes.Equal(t.rootHash, h) { + return fmt.Errorf("Incorrect rootHash") + } + return nil +} + +// Checks the rep invariants +func (t *node) validate() (count uint64, height int, err error) { + if t == nil { + return 0, 0, nil + } + if t.children[0] == nil { + if t.children[1] != nil { + return 0, 0, fmt.Errorf("Invalid Node: Node missing first child, but has second") + } + // Leaf node + return 1, 1, nil + } + + // Not a leaf node + count, height, err = t.children[0].validate() + if err != nil { + return + } + if t.children[1] != nil { + count2, height2, err2 := t.children[1].validate() + count += count2 + if err2 != nil { + return count, height, err2 + } + if height2 != height { + return count, height, fmt.Errorf("Invalid Node: height mismatch between children") + } + } + h := makeHash(t.children[0], t.children[1]) + if !bytes.Equal(h, t.label) { + return 0, 0, fmt.Errorf("Invalid Node: Node hash mismatch") + } + + height++ + return +} + +func rootHash(count uint64, data []byte) []byte { + h := hashFunc() + h.Reset() + h.Write(data) + binary.Write(h, binary.LittleEndian, count) + return h.Sum(make([]byte, 0)) +} + +func makeHash(left, right *node) []byte { + h := hashFunc() + h.Reset() + if left != nil { + h.Write(left.label) + if right != nil { + h.Write(right.label) + } + } + return h.Sum(make([]byte, 0)) +} + +// Returns the height of the tree containing count leaf nodes. +// This the number of nodes (including the final leaf) from the root to +// any leaf. +func GetHeight(count uint64) int { + if count == 0 { + return 0 + } + height := 0 + for count > (1 << uint(height)) { + height++ + } + return height + 1 +} + +// Build Binary Merkle Tree over data segments of segmentsize len with a specific hash func +// Return +// BMT - The BMT Representation of the data +// ROOT - BMT Root +// Count - Numers of leafs at the BMT +// error - if exist validation(-1) count(-2) ok(0) +func BuildBMT(h Hasher, data []byte, segmentsize int) (bmt *BTree, roor *Root, count int, errorcode int) { + blocks := splitData(data, segmentsize) + hashFunc = h + leafcount := len(blocks) + tree := Build(blocks) + err := tree.Validate() + if err != nil { + return nil, nil, 0, -1 + } + if tree.Count() != uint64(leafcount) { + return nil, nil, 0, -2 + } + + return tree, &Root{uint64(leafcount), tree.Root()}, leafcount, 0 + //r := Root{uint64(count), tree.Root()} + +} + +// Build a tree +func Build(data [][]byte) *BTree { + count := uint64(len(data)) + height := GetHeight(count) + node, leftOverData := buildNode(data, height) + if len(leftOverData) != 0 { + panic("Build failed to consume all data") + } + rootLabel := make([]byte, 0) + if height > 0 { + rootLabel = node.label + } + hash := rootHash(count, rootLabel) + t := BTree{count, node, hash} + return &t +} + +// returns a node and the left over data not used by it +func buildNode(data [][]byte, height int) (*node, [][]byte) { + if height == 0 || len(data) == 0 { + return nil, data + } + if height == 1 { + // leaf + return &node{label: data[0]}, data[1:] + } + n0, data := buildNode(data, height-1) + n1, data := buildNode(data, height-1) + + hash := makeHash(n0, n1) + return &node{label: hash, children: [2]*node{n0, n1}}, data +} + +func splitData(data []byte, size int) [][]byte { + /* Splits data into an array of slices of len(size) */ + count := len(data) / size + blocks := make([][]byte, 0, count) + for i := 0; i < count; i++ { + block := data[i*size : (i+1)*size] + blocks = append(blocks, block) + } + if len(data)%size != 0 { + blocks = append(blocks, data[len(blocks)*size:]) + } + return blocks +} + +// Return a [][]byte needed to prove the inclusion of the item at the passed index +// The payload of the item at index is the first value in the proof +func (t *BTree) InclusionProof(index int) [][]byte { + if uint64(index) >= t.count { + panic("Invalid index: too large") + } + if index < 0 { + panic("Invalid index: negative") + } + h := GetHeight(t.count) + fmt.Println(h) + return proveNode(h, t.root, index) +} + +func proveNode(height int, n *node, index int) [][]byte { + if height == 1 { + if index != 0 { + panic("Invalid index: non 0 for final node") + } + return [][]byte{n.label} + } + childIndex := index >> uint(height-2) + nextIndex := index & (^(1 << uint(height-2))) + b := proveNode(height-1, n.children[childIndex], nextIndex) + otherChildIndex := (childIndex + 1) % 2 + if n.children[otherChildIndex] != nil { + b = append(b, n.children[otherChildIndex].label) + } + return b +} + +// The Root of a merkle tree for a client that does not store the tree +type Root struct { + Count uint64 + Base []byte +} + +// Proves the inclusion of an element at the given index with the value thats the first entry in proof +func (r *Root) CheckProof(h Hasher, proof [][]byte, index int) bool { + hashFunc = h + t_height := GetHeight(r.Count) + root, ok := checkNode(t_height, proof, uint64(index), r.Count) + base := rootHash(r.Count, root) + return ok && bytes.Equal(r.Base, base) +} + +func checkNode(height int, proof [][]byte, index, count uint64) ([]byte, bool) { + if len(proof) == 0 { + fmt.Println("Empty") + return nil, false + } + if count <= index { + fmt.Println("bad count", count, index) + return nil, false + } + + if height == 1 { + if index != 0 || len(proof) != 1 { + fmt.Println("BAD", index, proof) + return nil, false + } + return proof[0], true + } + + childIndex := index >> uint(height-2) + mask := uint64(^(1 << uint(height-2))) + nextIndex := index & mask + + var data []byte + var ok bool + + h := hashFunc() + h.Reset() + // h:=hashFunc.New() + var nextCount uint64 + last := len(proof) - 1 + if childIndex == 1 { + nextCount = count & mask + h.Write(proof[last]) + data, ok = checkNode(height-1, proof[:last], nextIndex, nextCount) + h.Write(data) + } else { + nextCount = count + if count > ^mask { + nextCount = ^mask + } + if count == nextCount { + data, ok = checkNode(height-1, proof, nextIndex, nextCount) + h.Write(data) + } else { + data, ok = checkNode(height-1, proof[:last], nextIndex, nextCount) + h.Write(data) + h.Write(proof[last]) + } + } + + hash := h.Sum(make([]byte, 0)) + return hash, ok +} diff --git a/swarm/storage/binarymerkle_test.go b/swarm/storage/binarymerkle_test.go new file mode 100644 index 0000000000..98ddc13fbe --- /dev/null +++ b/swarm/storage/binarymerkle_test.go @@ -0,0 +1,123 @@ +package storage + +import ( + "fmt" + "io/ioutil" + "testing" + + "github.com/ethereum/go-ethereum/crypto/sha3" +) + +func TestGetHeight(t *testing.T) { + data := [][2]int{ + {0, 0}, + {1, 1}, + {2, 2}, + {3, 3}, + {4, 3}, + {255, 9}, + {256, 9}, + {257, 10}, + } + for _, v := range data { + h := GetHeight(uint64(v[0])) + if !(v[1] == h) { + t.Errorf("GetHeight(%d)!=%d (was %d)", v[0], v[1], h) + } + } +} + +func TestGetHeight2(t *testing.T) { + for i := 1; i < 1000; i++ { + h := GetHeight(uint64(i)) + upperBound := 1 << uint(h-1) + lowerBound := (1 << uint(h-2)) + 1 + if i < lowerBound { + t.Errorf("GetHeight(%d) too high: %d", i, h) + } + if i > upperBound { + t.Errorf("GetHeight(%d) too low: %d", i, h) + } + } +} + +func TestBuildBMT(t *testing.T) { + + // Grab some data to make the tree out of, and partition + data, err := ioutil.ReadFile("binarymerkle_test.go") // assume testdata exists + if err != nil { + fmt.Println(err) + return + } + + tree, r, count, err1 := BuildBMT(sha3.NewKeccak256, data, 32) + + switch err1 { + case -1: + + t.Errorf("BMT Validation error") + return + case -2: + t.Errorf("BMT leaf count validation error") + return + case 0: + fmt.Println("Build BMT OK") + } + + fmt.Println(tree.Root()) + + for i := 0; i < count; i++ { + p := tree.InclusionProof(i) + + fmt.Println(p) + + ok := r.CheckProof(sha3.NewKeccak256, p, i) + if !ok { + t.Errorf("proof %d failed", i) + } + } +} + +func TestBuildBMT2(t *testing.T) { + + // Grab some data to make the tree out of, and partition + data, err := ioutil.ReadFile("binarymerkle_test.go") // assume testdata exists + if err != nil { + fmt.Println(err) + return + } + + fmt.Println(len(data)) + blocks := splitData(data, 32) + + count := len(blocks) + + // t.Errorf("GetCount() != %d (was )", count) + + tree := Build(blocks) + err1 := tree.Validate() + if err1 != nil { + t.Errorf("%s", err1) + } + if tree.Count() != uint64(count) { + t.Errorf("GetCount() != %d (was %d)", count, tree.Count()) + } + + r := Root{uint64(count), tree.Root()} + + fmt.Println(tree.Root()) + + for i := 0; i < count; i++ { + p := tree.InclusionProof(i) + + fmt.Println(p) + + ok := r.CheckProof(sha3.NewKeccak256, p, i) + if !ok { + t.Errorf("proof %d failed", i) + } + } + //t.Errorf("proof ok") + // TODO: check wrong proofs fail + +} From 44a9a39c846e365222cabcdff33aff0919a64a9d Mon Sep 17 00:00:00 2001 From: Oren Date: Fri, 24 Mar 2017 07:49:35 +0300 Subject: [PATCH 2/7] swarm/storage add hash.Hash interface for BMT . Integrate with chunker.go --- swarm/storage/binarymerkle.go | 69 ++++++++++++++++++++++++++++++ swarm/storage/binarymerkle_test.go | 36 ++++++++++++++-- swarm/storage/chunker.go | 3 +- swarm/storage/types.go | 2 + 4 files changed, 106 insertions(+), 4 deletions(-) diff --git a/swarm/storage/binarymerkle.go b/swarm/storage/binarymerkle.go index 0c31db1698..972dea17cc 100644 --- a/swarm/storage/binarymerkle.go +++ b/swarm/storage/binarymerkle.go @@ -6,13 +6,21 @@ import ( "bytes" _ "crypto/sha256" "encoding/binary" + "errors" "fmt" + "hash" + "io" "github.com/ethereum/go-ethereum/crypto/sha3" ) var hashFunc Hasher = sha3.NewKeccak256 //default hasher +type state struct { + btree BTree + root Root +} + // A merkle tree for a user that stores the entire tree // Specifically this tree is left a leaning balanced binary tree // Where each node holds the hash of its leaves @@ -313,3 +321,64 @@ func checkNode(height int, proof [][]byte, index, count uint64) ([]byte, bool) { hash := h.Sum(make([]byte, 0)) return hash, ok } + +// ShakeHash defines the interface to hash functions that +// support arbitrary-length output. +type BMTHash interface { + // Write absorbs more data into the hash's state. It panics if input is + // written to it after output has been read from it. + io.Writer + + // Read reads more output from the hash; reading affects the hash's + // state. (ShakeHash.Read is thus very different from Hash.Sum) + // It never returns an error. + io.Reader + + // Clone returns a copy of the ShakeHash in its current state. + Clone() BMTHash + + // Reset resets the ShakeHash to its initial state. + Reset() +} + +// Reset clears the internal state by zeroing the sponge state and +func (d *state) Reset() { + d.root = Root{Count: 0, Base: nil} + d.btree = BTree{count: 0, root: nil, rootHash: nil} +} + +// Write absorbs more data into the hash's state. It produces an error +// if more data is written to the ShakeHash after writing +func (d *state) Write(p []byte) (written int, err error) { + tree, r, count, err1 := BuildBMT(hashFunc, p, 32) + d.btree = *tree + d.root = *r + + if err1 != 0 { + err = errors.New("bmt write error") + } + + return count, err +} + +func (d *state) Get(p []byte) (written int) { + return 3 +} + +// Sum return the root hash of the BMT +func (d *state) Sum(in []byte) []byte { + return d.root.Base +} + +// BlockSize returns the rate of sponge underlying this hash function. +func (d *state) BlockSize() int { return 0 } + +// Size returns the output size of the hash function in bytes. +func (d *state) Size() int { return 32 } + +// NewBMTSHA3 creates a new BMT hash +func NewBMTSHA3() hash.Hash { + tmpbtree := BTree{count: 0, root: nil, rootHash: nil} + troot := Root{Count: 0, Base: nil} + return &state{btree: tmpbtree, root: troot} +} diff --git a/swarm/storage/binarymerkle_test.go b/swarm/storage/binarymerkle_test.go index 98ddc13fbe..28faea5032 100644 --- a/swarm/storage/binarymerkle_test.go +++ b/swarm/storage/binarymerkle_test.go @@ -41,6 +41,35 @@ func TestGetHeight2(t *testing.T) { } } +func TestBuildBMT3(t *testing.T) { + + // Grab some data to make the tree out of, and partition + data, err := ioutil.ReadFile("binarymerkle_test.go") // assume testdata exists + if err != nil { + fmt.Println(err) + return + } + var thashFunc = MakeHashFunc("BMTSHA3") + var h = thashFunc() + h.Reset() + h.Write(data) + var key = h.Sum(nil) + + fmt.Println(key) + + // for i := 0; i < count; i++ { + // p := tree.InclusionProof(i) + // + // fmt.Println(p) + // + // ok := r.CheckProof(sha3.NewKeccak256, p, i) + // if !ok { + // t.Errorf("proof %d failed", i) + // } + // } + // fmt.Println("done") +} + func TestBuildBMT(t *testing.T) { // Grab some data to make the tree out of, and partition @@ -50,7 +79,7 @@ func TestBuildBMT(t *testing.T) { return } - tree, r, count, err1 := BuildBMT(sha3.NewKeccak256, data, 32) + tree, r, count, err1 := BuildBMT(sha3.NewKeccak256, data, 1) switch err1 { case -1: @@ -61,7 +90,7 @@ func TestBuildBMT(t *testing.T) { t.Errorf("BMT leaf count validation error") return case 0: - fmt.Println("Build BMT OK") + fmt.Println("Build BMT OK ", count) } fmt.Println(tree.Root()) @@ -76,6 +105,7 @@ func TestBuildBMT(t *testing.T) { t.Errorf("proof %d failed", i) } } + fmt.Println("done") } func TestBuildBMT2(t *testing.T) { @@ -88,7 +118,7 @@ func TestBuildBMT2(t *testing.T) { } fmt.Println(len(data)) - blocks := splitData(data, 32) + blocks := splitData(data, 2) count := len(blocks) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index d55875369d..a2eda3fdf7 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -51,7 +51,8 @@ data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1} */ const ( - defaultHash = "SHA3" // http://golang.org/pkg/hash/#Hash + //defaultHash = "SHA3" + defaultHash = "BMTSHA3" // http://golang.org/pkg/hash/#Hash // defaultHash = "SHA256" // http://golang.org/pkg/hash/#Hash defaultBranches int64 = 128 // hashSize int64 = hasherfunc.New().Size() // hasher knows about its own length in bytes diff --git a/swarm/storage/types.go b/swarm/storage/types.go index cc5ded931d..1afb82e48e 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -83,6 +83,8 @@ func MakeHashFunc(hash string) Hasher { return crypto.SHA256.New case "SHA3": return sha3.NewKeccak256 + case "BMTSHA3": + return NewBMTSHA3 } return nil } From fe85196dd634d88d2a672fc907f38d3da2f7a43b Mon Sep 17 00:00:00 2001 From: Oren Date: Sun, 26 Mar 2017 12:00:20 +0300 Subject: [PATCH 3/7] Use errors (instead of error codes) --- swarm/storage/binarymerkle.go | 64 +++++++++++------------ swarm/storage/binarymerkle_test.go | 82 +++++++----------------------- 2 files changed, 49 insertions(+), 97 deletions(-) diff --git a/swarm/storage/binarymerkle.go b/swarm/storage/binarymerkle.go index 972dea17cc..edc5e4faaf 100644 --- a/swarm/storage/binarymerkle.go +++ b/swarm/storage/binarymerkle.go @@ -58,9 +58,9 @@ func (t BTree) Root() []byte { // if incorrectly built or modified. // Checks the rep invariants func (t BTree) Validate() error { - count, height, error := t.root.validate() - if error != nil { - return error + count, height, err := t.root.validate() + if err != nil { + return err } if count != t.count { return fmt.Errorf("Incorrect count. Was %d, should be %d", t.count, count) @@ -122,7 +122,7 @@ func rootHash(count uint64, data []byte) []byte { h.Reset() h.Write(data) binary.Write(h, binary.LittleEndian, count) - return h.Sum(make([]byte, 0)) + return h.Sum(nil) } func makeHash(left, right *node) []byte { @@ -156,21 +156,21 @@ func GetHeight(count uint64) int { // BMT - The BMT Representation of the data // ROOT - BMT Root // Count - Numers of leafs at the BMT -// error - if exist validation(-1) count(-2) ok(0) -func BuildBMT(h Hasher, data []byte, segmentsize int) (bmt *BTree, roor *Root, count int, errorcode int) { +// error - +func BuildBMT(h Hasher, data []byte, segmentsize int) (bmt *BTree, roor *Root, count int, err error) { blocks := splitData(data, segmentsize) hashFunc = h leafcount := len(blocks) tree := Build(blocks) - err := tree.Validate() + err = tree.Validate() if err != nil { - return nil, nil, 0, -1 + return nil, nil, 0, errors.New("Validation error") } if tree.Count() != uint64(leafcount) { - return nil, nil, 0, -2 + return nil, nil, 0, errors.New("Validation count error") } - return tree, &Root{uint64(leafcount), tree.Root()}, leafcount, 0 + return tree, &Root{uint64(leafcount), tree.Root()}, leafcount, nil //r := Root{uint64(count), tree.Root()} } @@ -260,30 +260,29 @@ type Root struct { } // Proves the inclusion of an element at the given index with the value thats the first entry in proof -func (r *Root) CheckProof(h Hasher, proof [][]byte, index int) bool { +func (r *Root) CheckProof(h Hasher, proof [][]byte, index int) (bool, error) { hashFunc = h - t_height := GetHeight(r.Count) - root, ok := checkNode(t_height, proof, uint64(index), r.Count) + theight := GetHeight(r.Count) + var root, ok, err = checkNode(theight, proof, uint64(index), r.Count) base := rootHash(r.Count, root) - return ok && bytes.Equal(r.Base, base) + return ok && bytes.Equal(r.Base, base), err } -func checkNode(height int, proof [][]byte, index, count uint64) ([]byte, bool) { +func checkNode(height int, proof [][]byte, index uint64, count uint64) (hash []byte, ok bool, err error) { if len(proof) == 0 { - fmt.Println("Empty") - return nil, false + return nil, false, errors.New("checkNode : proof is empty") } if count <= index { fmt.Println("bad count", count, index) - return nil, false + return nil, false, fmt.Errorf("bad count %d at index %d", count, index) } if height == 1 { if index != 0 || len(proof) != 1 { fmt.Println("BAD", index, proof) - return nil, false + return nil, false, fmt.Errorf("BAD %d %d", index, proof) } - return proof[0], true + return proof[0], true, nil } childIndex := index >> uint(height-2) @@ -291,7 +290,7 @@ func checkNode(height int, proof [][]byte, index, count uint64) ([]byte, bool) { nextIndex := index & mask var data []byte - var ok bool + //var ok bool h := hashFunc() h.Reset() @@ -301,7 +300,7 @@ func checkNode(height int, proof [][]byte, index, count uint64) ([]byte, bool) { if childIndex == 1 { nextCount = count & mask h.Write(proof[last]) - data, ok = checkNode(height-1, proof[:last], nextIndex, nextCount) + data, ok, err = checkNode(height-1, proof[:last], nextIndex, nextCount) h.Write(data) } else { nextCount = count @@ -309,39 +308,38 @@ func checkNode(height int, proof [][]byte, index, count uint64) ([]byte, bool) { nextCount = ^mask } if count == nextCount { - data, ok = checkNode(height-1, proof, nextIndex, nextCount) + data, ok, err = checkNode(height-1, proof, nextIndex, nextCount) h.Write(data) } else { - data, ok = checkNode(height-1, proof[:last], nextIndex, nextCount) + data, ok, err = checkNode(height-1, proof[:last], nextIndex, nextCount) h.Write(data) h.Write(proof[last]) } } - hash := h.Sum(make([]byte, 0)) - return hash, ok + hash = h.Sum(make([]byte, 0)) + return hash, ok, nil } -// ShakeHash defines the interface to hash functions that -// support arbitrary-length output. +// BMTHash defines the interface to hash functions that type BMTHash interface { // Write absorbs more data into the hash's state. It panics if input is // written to it after output has been read from it. io.Writer // Read reads more output from the hash; reading affects the hash's - // state. (ShakeHash.Read is thus very different from Hash.Sum) + // state. // It never returns an error. io.Reader - // Clone returns a copy of the ShakeHash in its current state. + // Clone returns a copy of the BMTHash in its current state. Clone() BMTHash - // Reset resets the ShakeHash to its initial state. + // Reset resets the BMTHash to its initial state. Reset() } -// Reset clears the internal state by zeroing the sponge state and +// Reset clears the internal state func (d *state) Reset() { d.root = Root{Count: 0, Base: nil} d.btree = BTree{count: 0, root: nil, rootHash: nil} @@ -354,7 +352,7 @@ func (d *state) Write(p []byte) (written int, err error) { d.btree = *tree d.root = *r - if err1 != 0 { + if err1 != nil { err = errors.New("bmt write error") } diff --git a/swarm/storage/binarymerkle_test.go b/swarm/storage/binarymerkle_test.go index 28faea5032..bfd7650219 100644 --- a/swarm/storage/binarymerkle_test.go +++ b/swarm/storage/binarymerkle_test.go @@ -2,53 +2,17 @@ package storage import ( "fmt" - "io/ioutil" "testing" "github.com/ethereum/go-ethereum/crypto/sha3" ) -func TestGetHeight(t *testing.T) { - data := [][2]int{ - {0, 0}, - {1, 1}, - {2, 2}, - {3, 3}, - {4, 3}, - {255, 9}, - {256, 9}, - {257, 10}, - } - for _, v := range data { - h := GetHeight(uint64(v[0])) - if !(v[1] == h) { - t.Errorf("GetHeight(%d)!=%d (was %d)", v[0], v[1], h) - } - } -} - -func TestGetHeight2(t *testing.T) { - for i := 1; i < 1000; i++ { - h := GetHeight(uint64(i)) - upperBound := 1 << uint(h-1) - lowerBound := (1 << uint(h-2)) + 1 - if i < lowerBound { - t.Errorf("GetHeight(%d) too high: %d", i, h) - } - if i > upperBound { - t.Errorf("GetHeight(%d) too low: %d", i, h) - } - } -} - func TestBuildBMT3(t *testing.T) { // Grab some data to make the tree out of, and partition - data, err := ioutil.ReadFile("binarymerkle_test.go") // assume testdata exists - if err != nil { - fmt.Println(err) - return - } + data := make([]byte, 4096) + tdata := testDataReader(4096) + tdata.Read(data) var thashFunc = MakeHashFunc("BMTSHA3") var h = thashFunc() h.Reset() @@ -73,35 +37,28 @@ func TestBuildBMT3(t *testing.T) { func TestBuildBMT(t *testing.T) { // Grab some data to make the tree out of, and partition - data, err := ioutil.ReadFile("binarymerkle_test.go") // assume testdata exists - if err != nil { - fmt.Println(err) - return - } - + data := make([]byte, 4096) + tdata := testDataReader(4096) + tdata.Read(data) tree, r, count, err1 := BuildBMT(sha3.NewKeccak256, data, 1) - switch err1 { - case -1: - - t.Errorf("BMT Validation error") + if err1 != nil { + fmt.Println(err1) return - case -2: - t.Errorf("BMT leaf count validation error") - return - case 0: - fmt.Println("Build BMT OK ", count) } fmt.Println(tree.Root()) + //var ok bool + for i := 0; i < count; i++ { p := tree.InclusionProof(i) fmt.Println(p) - ok := r.CheckProof(sha3.NewKeccak256, p, i) - if !ok { + ok, err := r.CheckProof(sha3.NewKeccak256, p, i) + + if !ok || (err != nil) { t.Errorf("proof %d failed", i) } } @@ -110,12 +67,9 @@ func TestBuildBMT(t *testing.T) { func TestBuildBMT2(t *testing.T) { - // Grab some data to make the tree out of, and partition - data, err := ioutil.ReadFile("binarymerkle_test.go") // assume testdata exists - if err != nil { - fmt.Println(err) - return - } + data := make([]byte, 4096) + tdata := testDataReader(4096) + tdata.Read(data) fmt.Println(len(data)) blocks := splitData(data, 2) @@ -142,8 +96,8 @@ func TestBuildBMT2(t *testing.T) { fmt.Println(p) - ok := r.CheckProof(sha3.NewKeccak256, p, i) - if !ok { + ok, err := r.CheckProof(sha3.NewKeccak256, p, i) + if !ok || (err != nil) { t.Errorf("proof %d failed", i) } } From bef40cd1ef355fea64a12e2dcbdbe5addb9499db Mon Sep 17 00:00:00 2001 From: Oren Date: Sun, 26 Mar 2017 12:12:55 +0300 Subject: [PATCH 4/7] cleanup --- swarm/storage/binarymerkle.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/swarm/storage/binarymerkle.go b/swarm/storage/binarymerkle.go index edc5e4faaf..2f828c4703 100644 --- a/swarm/storage/binarymerkle.go +++ b/swarm/storage/binarymerkle.go @@ -345,8 +345,7 @@ func (d *state) Reset() { d.btree = BTree{count: 0, root: nil, rootHash: nil} } -// Write absorbs more data into the hash's state. It produces an error -// if more data is written to the ShakeHash after writing +// Write absorbs more data into the hash's state. func (d *state) Write(p []byte) (written int, err error) { tree, r, count, err1 := BuildBMT(hashFunc, p, 32) d.btree = *tree @@ -359,10 +358,6 @@ func (d *state) Write(p []byte) (written int, err error) { return count, err } -func (d *state) Get(p []byte) (written int) { - return 3 -} - // Sum return the root hash of the BMT func (d *state) Sum(in []byte) []byte { return d.root.Base From aacecda1a43f1d0028430e52458ecc56fd559dff Mon Sep 17 00:00:00 2001 From: Oren Date: Mon, 27 Mar 2017 13:55:56 +0300 Subject: [PATCH 5/7] parellel computing --- swarm/storage/binarymerkle.go | 146 +++++++++++++++++++++++++++-- swarm/storage/binarymerkle_test.go | 75 ++++++--------- 2 files changed, 167 insertions(+), 54 deletions(-) diff --git a/swarm/storage/binarymerkle.go b/swarm/storage/binarymerkle.go index 2f828c4703..12e0e96ea7 100644 --- a/swarm/storage/binarymerkle.go +++ b/swarm/storage/binarymerkle.go @@ -10,6 +10,7 @@ import ( "fmt" "hash" "io" + "sync" "github.com/ethereum/go-ethereum/crypto/sha3" ) @@ -21,6 +22,11 @@ type state struct { root Root } +const ( + // max number of leafs on 4096 chunk len in 32 bytes segments + mprocessors = 128 +) + // A merkle tree for a user that stores the entire tree // Specifically this tree is left a leaning balanced binary tree // Where each node holds the hash of its leaves @@ -45,6 +51,20 @@ type node struct { // else if children[1] is nil, label=hash(children[0].label) } +type jobparam struct { + data [][]byte + height int + n0 *node + n1 *node + id int +} + +type jobresult struct { + n *node + data [][]byte + id int +} + func (t BTree) Count() uint64 { return t.count } @@ -158,10 +178,12 @@ func GetHeight(count uint64) int { // Count - Numers of leafs at the BMT // error - func BuildBMT(h Hasher, data []byte, segmentsize int) (bmt *BTree, roor *Root, count int, err error) { - blocks := splitData(data, segmentsize) hashFunc = h - leafcount := len(blocks) - tree := Build(blocks) + leafcount := len(data) / segmentsize + if len(data)%segmentsize != 0 { + leafcount++ + } + tree := BuildParralel(data, segmentsize) err = tree.Validate() if err != nil { return nil, nil, 0, errors.New("Validation error") @@ -171,7 +193,28 @@ func BuildBMT(h Hasher, data []byte, segmentsize int) (bmt *BTree, roor *Root, c } return tree, &Root{uint64(leafcount), tree.Root()}, leafcount, nil - //r := Root{uint64(count), tree.Root()} + +} + +func processor(pend *sync.WaitGroup, id int, tasks <-chan jobparam, results chan<- jobresult) { + + defer pend.Done() + for task := range tasks { + + var res jobresult + + switch task.height { + case 0: + res = jobresult{nil, task.data, task.id} + case 1: + res = jobresult{&node{label: task.data[0]}, nil, task.id} + default: + hash := makeHash(task.n0, task.n1) + res = jobresult{&node{label: hash, children: [2]*node{task.n0, task.n1}}, task.data, task.id} + } + results <- res + pend.Done() + } } @@ -179,6 +222,7 @@ func BuildBMT(h Hasher, data []byte, segmentsize int) (bmt *BTree, roor *Root, c func Build(data [][]byte) *BTree { count := uint64(len(data)) height := GetHeight(count) + node, leftOverData := buildNode(data, height) if len(leftOverData) != 0 { panic("Build failed to consume all data") @@ -192,6 +236,96 @@ func Build(data [][]byte) *BTree { return &t } +// Build a tree parralel computation +// We build it level by level ..starting from the depper one and get up.. +func BuildParralel(data []byte, segmentsize int) *BTree { + count := len(data) / segmentsize + var height int + + if len(data)%segmentsize != 0 { + height = GetHeight(uint64(count + 1)) + + } else { + height = GetHeight(uint64(count)) + } + + jobs := make(chan jobparam, mprocessors) + results := make(chan jobresult, mprocessors) + + pend := new(sync.WaitGroup) + + for w := 1; w <= mprocessors; w++ { + pend.Add(1) + go processor(pend, 2, jobs, results) + } + /*build leafs*/ + var leaflevelcount = 1 << uint(height-1) + for k := 0; k < leaflevelcount; k++ { + datasegment := make([][]byte, 0, 1) + pend.Add(1) + if k*segmentsize+segmentsize > len(data) { + if (k*segmentsize + len(data)%segmentsize) > len(data) { + jobs <- jobparam{data: nil, height: 0, n0: nil, n1: nil, id: k} + } else { + datasegment = append(datasegment, data[k*segmentsize:k*segmentsize+len(data)%segmentsize]) + jobs <- jobparam{data: datasegment, height: 1, n0: nil, n1: nil, id: k} + } + } else { + datasegment = append(datasegment, data[k*segmentsize:(k+1)*segmentsize]) + jobs <- jobparam{data: datasegment, height: 1, n0: nil, n1: nil, id: k} + } + } + fmt.Println("1") + close(jobs) + pend.Wait() + + if len(data)%segmentsize != 0 { + count++ + } + restmp := make([]jobresult, leaflevelcount) + fmt.Println("2", leaflevelcount) + //Get computation results + for i := 0; i < leaflevelcount; i++ { + tres := <-results + restmp[tres.id] = tres + } + fmt.Println("3", height) + for i := 1; i < height; i++ { + jobs := make(chan jobparam, mprocessors) + for w := 1; w <= mprocessors; w++ { + pend.Add(1) + go processor(pend, 2, jobs, results) + } + var indexres = 0 + for j := 0; j < (1<>uint(i); j++ { + pend.Add(1) + jobs <- jobparam{n0: restmp[indexres].n, n1: restmp[indexres+1].n, data: restmp[indexres+1].data, height: 2, id: j} + indexres += 2 + } + close(jobs) + pend.Wait() + if i < height-1 { + for l := 0; l < (1<>uint(i); l++ { + tmpres := <-results + restmp[tmpres.id] = tmpres + } + } + } + rootLabel := make([]byte, 0) + var res jobresult + if leaflevelcount == 1 { + res = restmp[0] + } else { + res = <-results + } + if height > 0 { + rootLabel = res.n.label + } + hash := rootHash(uint64(count), rootLabel) + t := BTree{uint64(count), res.n, hash} + return &t +} + // returns a node and the left over data not used by it func buildNode(data [][]byte, height int) (*node, [][]byte) { if height == 0 || len(data) == 0 { @@ -222,7 +356,7 @@ func splitData(data []byte, size int) [][]byte { return blocks } -// Return a [][]byte needed to prove the inclusion of the item at the passed index +// Return a [][]byte needed to prove the gkf of the item at the passed index // The payload of the item at index is the first value in the proof func (t *BTree) InclusionProof(index int) [][]byte { if uint64(index) >= t.count { @@ -259,7 +393,7 @@ type Root struct { Base []byte } -// Proves the inclusion of an element at the given index with the value thats the first entry in proof +// Proves theof an element at the given index with the value thats the first entry in proof func (r *Root) CheckProof(h Hasher, proof [][]byte, index int) (bool, error) { hashFunc = h theight := GetHeight(r.Count) diff --git a/swarm/storage/binarymerkle_test.go b/swarm/storage/binarymerkle_test.go index bfd7650219..ec604db583 100644 --- a/swarm/storage/binarymerkle_test.go +++ b/swarm/storage/binarymerkle_test.go @@ -2,12 +2,15 @@ package storage import ( "fmt" + "log" "testing" + "time" "github.com/ethereum/go-ethereum/crypto/sha3" ) -func TestBuildBMT3(t *testing.T) { +//will test the hash (hash.Hash) interface +func TestBuildBMT2(t *testing.T) { // Grab some data to make the tree out of, and partition data := make([]byte, 4096) @@ -34,22 +37,39 @@ func TestBuildBMT3(t *testing.T) { // fmt.Println("done") } +func TestBuildBMTStress(t *testing.T) { + + for i := 1; i < 4096; i++ { + BuildBMTwithGivenDataLen(i, t) + } +} + func TestBuildBMT(t *testing.T) { + BuildBMTwithGivenDataLen(4096, t) +} + +func BuildBMTwithGivenDataLen(datalen int, t *testing.T) { // Grab some data to make the tree out of, and partition - data := make([]byte, 4096) - tdata := testDataReader(4096) + fmt.Println("dl", datalen) + data := make([]byte, datalen) + tdata := testDataReader(datalen) tdata.Read(data) - tree, r, count, err1 := BuildBMT(sha3.NewKeccak256, data, 1) + fmt.Println(data) + + start := time.Now() + + tree, r, count, err1 := BuildBMT(sha3.NewKeccak256, data, 32) + + elapsed := time.Since(start) + log.Printf("Binomial took %s", elapsed) if err1 != nil { fmt.Println(err1) return } - fmt.Println(tree.Root()) - - //var ok bool + fmt.Println(tree.Root(), count) for i := 0; i < count; i++ { p := tree.InclusionProof(i) @@ -64,44 +84,3 @@ func TestBuildBMT(t *testing.T) { } fmt.Println("done") } - -func TestBuildBMT2(t *testing.T) { - - data := make([]byte, 4096) - tdata := testDataReader(4096) - tdata.Read(data) - - fmt.Println(len(data)) - blocks := splitData(data, 2) - - count := len(blocks) - - // t.Errorf("GetCount() != %d (was )", count) - - tree := Build(blocks) - err1 := tree.Validate() - if err1 != nil { - t.Errorf("%s", err1) - } - if tree.Count() != uint64(count) { - t.Errorf("GetCount() != %d (was %d)", count, tree.Count()) - } - - r := Root{uint64(count), tree.Root()} - - fmt.Println(tree.Root()) - - for i := 0; i < count; i++ { - p := tree.InclusionProof(i) - - fmt.Println(p) - - ok, err := r.CheckProof(sha3.NewKeccak256, p, i) - if !ok || (err != nil) { - t.Errorf("proof %d failed", i) - } - } - //t.Errorf("proof ok") - // TODO: check wrong proofs fail - -} From d56327b887684d5a6b99a348bbb38b87c24848c2 Mon Sep 17 00:00:00 2001 From: Oren Date: Mon, 27 Mar 2017 14:11:45 +0300 Subject: [PATCH 6/7] check data len for 0 --- swarm/storage/binarymerkle.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/swarm/storage/binarymerkle.go b/swarm/storage/binarymerkle.go index 12e0e96ea7..c33fe3d1a9 100644 --- a/swarm/storage/binarymerkle.go +++ b/swarm/storage/binarymerkle.go @@ -178,6 +178,9 @@ func GetHeight(count uint64) int { // Count - Numers of leafs at the BMT // error - func BuildBMT(h Hasher, data []byte, segmentsize int) (bmt *BTree, roor *Root, count int, err error) { + if len(data) == 0 { + return nil, nil, 0, errors.New("data length is 0 ") + } hashFunc = h leafcount := len(data) / segmentsize if len(data)%segmentsize != 0 { From 185122326826c4cd29e18a3f74c9081855a18eaa Mon Sep 17 00:00:00 2001 From: Oren Date: Fri, 7 Apr 2017 12:52:06 +0300 Subject: [PATCH 7/7] swarm/storage - Inclusion proof for any given offset and length within a specific chunk -Performance improvements --- swarm/storage/binarymerkle.go | 410 +++++++++++++++++------------ swarm/storage/binarymerkle_test.go | 214 +++++++++++---- 2 files changed, 393 insertions(+), 231 deletions(-) diff --git a/swarm/storage/binarymerkle.go b/swarm/storage/binarymerkle.go index c33fe3d1a9..9daca3e008 100644 --- a/swarm/storage/binarymerkle.go +++ b/swarm/storage/binarymerkle.go @@ -5,28 +5,25 @@ package storage import ( "bytes" _ "crypto/sha256" - "encoding/binary" "errors" "fmt" "hash" "io" - "sync" "github.com/ethereum/go-ethereum/crypto/sha3" ) var hashFunc Hasher = sha3.NewKeccak256 //default hasher +const ( + segmentsize int = 32 +) + type state struct { btree BTree root Root } -const ( - // max number of leafs on 4096 chunk len in 32 bytes segments - mprocessors = 128 -) - // A merkle tree for a user that stores the entire tree // Specifically this tree is left a leaning balanced binary tree // Where each node holds the hash of its leaves @@ -36,6 +33,7 @@ type BTree struct { count uint64 root *node rootHash []byte + chunklen int //hashFunc Hasher } @@ -52,17 +50,15 @@ type node struct { } type jobparam struct { - data [][]byte - height int - n0 *node - n1 *node - id int + //data [][]byte + n0 *node + n1 *node + id int } type jobresult struct { - n *node - data [][]byte - id int + n *node + id int } func (t BTree) Count() uint64 { @@ -141,20 +137,21 @@ func rootHash(count uint64, data []byte) []byte { h := hashFunc() h.Reset() h.Write(data) - binary.Write(h, binary.LittleEndian, count) + //binary.Write(h, binary.LittleEndian, count) return h.Sum(nil) } func makeHash(left, right *node) []byte { h := hashFunc() h.Reset() + if left != nil { h.Write(left.label) if right != nil { h.Write(right.label) } } - return h.Sum(make([]byte, 0)) + return h.Sum(nil) } // Returns the height of the tree containing count leaf nodes. @@ -176,161 +173,173 @@ func GetHeight(count uint64) int { // BMT - The BMT Representation of the data // ROOT - BMT Root // Count - Numers of leafs at the BMT -// error - -func BuildBMT(h Hasher, data []byte, segmentsize int) (bmt *BTree, roor *Root, count int, err error) { - if len(data) == 0 { +// err +// The bmt computation is done in parallel by deviding the tree to subtree . +// (each sub tree is calculated in parallel using goroutine ) and then merge the results in parallel and get the tree. +// The paralel merging is done by creating seperate channel for each node and make it to wait(on a seperate go routine) for the calculation of +// its left and right childerens. +func BuildBMT(h Hasher, data []byte, validate bool) (bmt *BTree, roor *Root, count int, err error) { + + if (len(data) & (len(data) - 1)) == 0 { //check if power of 2 + return buildBMTfaster(h, data, validate) + } else { + return buildBMTfast(h, data, validate) + } +} + +//This function assume its data len value is a power of 2 +func buildBMTfaster(h Hasher, data []byte, validate bool) (bmt *BTree, roor *Root, count int, err error) { + + datalen := len(data) + if datalen == 0 { return nil, nil, 0, errors.New("data length is 0 ") } hashFunc = h - leafcount := len(data) / segmentsize - if len(data)%segmentsize != 0 { + leafcount := datalen / segmentsize + if datalen%segmentsize != 0 { leafcount++ } - tree := BuildParralel(data, segmentsize) - err = tree.Validate() - if err != nil { - return nil, nil, 0, errors.New("Validation error") - } - if tree.Count() != uint64(leafcount) { - return nil, nil, 0, errors.New("Validation count error") + var rootnode *node + var subtreescount = 4 + //setting the subtreescount to 4 yield the best benchmarks results. + if leafcount < 4 { + subtreescount = 2 } + if leafcount > 1 { + subtreesize := datalen / subtreescount + subtreeleafcount := subtreesize / segmentsize + if subtreesize%segmentsize != 0 { + subtreeleafcount++ + } + subtreeheight := GetHeight(uint64(subtreeleafcount)) + height := GetHeight(uint64(subtreescount)) + results := make([]chan *node, (1 << uint(height))) //array of channels for each node + //start := time.Now() + + for i := 0; i < subtreescount; i++ { + results[i] = make(chan *node) + results[subtreescount+i] = make(chan *node) + go func(subdata []byte, index int) { + subtreerootnode, _ := buildNode(subdata, subtreeheight) + results[index] <- subtreerootnode + }(data[i*subtreesize:(i+1)*subtreesize], i) + } + for i := 0; i < subtreescount-1; i++ { + go func(index int, resultindex int) { + var leftnode, rightnode *node + select { + case leftnode = <-results[index]: + rightnode = <-results[index+1] + case rightnode = <-results[index+1]: + leftnode = <-results[index] + } + results[resultindex] <- &node{label: makeHash(leftnode, rightnode), + children: [2]*node{leftnode, rightnode}} + }(i*2, subtreescount+i) + } + + rootnode = <-results[(1< 1 { + subtreesize := leafcount / subtreescount + subtreeheight := GetHeight(uint64(subtreesize)) + height := GetHeight(uint64(subtreescount)) + results := make([]chan *node, (1 << uint(height))) //array of channels for each node + + for i := 0; i < subtreescount; i++ { + results[i] = make(chan *node) + results[subtreescount+i] = make(chan *node) + go func(subdata [][]byte, index int) { + subtreerootnode, _ := buildNode2(subdata, subtreeheight) + results[index] <- subtreerootnode + }(blocks[i*(leafcount/subtreescount):(i+1)*(leafcount/subtreescount)], i) + } + + for i := 0; i < subtreescount-1; i++ { + go func(index int, resultindex int) { + var leftnode, rightnode *node + select { + case leftnode = <-results[index]: + rightnode = <-results[index+1] + case rightnode = <-results[index+1]: + leftnode = <-results[index] + } + results[resultindex] <- &node{label: makeHash(leftnode, rightnode), + children: [2]*node{leftnode, rightnode}} + }(i*2, subtreescount+i) + + } + + rootnode = <-results[(1< 0 { - rootLabel = node.label - } - hash := rootHash(count, rootLabel) - t := BTree{count, node, hash} - return &t -} - -// Build a tree parralel computation -// We build it level by level ..starting from the depper one and get up.. -func BuildParralel(data []byte, segmentsize int) *BTree { - count := len(data) / segmentsize - var height int - - if len(data)%segmentsize != 0 { - height = GetHeight(uint64(count + 1)) - - } else { - height = GetHeight(uint64(count)) - } - - jobs := make(chan jobparam, mprocessors) - results := make(chan jobresult, mprocessors) - - pend := new(sync.WaitGroup) - - for w := 1; w <= mprocessors; w++ { - pend.Add(1) - go processor(pend, 2, jobs, results) - } - /*build leafs*/ - var leaflevelcount = 1 << uint(height-1) - for k := 0; k < leaflevelcount; k++ { - datasegment := make([][]byte, 0, 1) - pend.Add(1) - if k*segmentsize+segmentsize > len(data) { - if (k*segmentsize + len(data)%segmentsize) > len(data) { - jobs <- jobparam{data: nil, height: 0, n0: nil, n1: nil, id: k} - } else { - datasegment = append(datasegment, data[k*segmentsize:k*segmentsize+len(data)%segmentsize]) - jobs <- jobparam{data: datasegment, height: 1, n0: nil, n1: nil, id: k} - } - } else { - datasegment = append(datasegment, data[k*segmentsize:(k+1)*segmentsize]) - jobs <- jobparam{data: datasegment, height: 1, n0: nil, n1: nil, id: k} - } - } - fmt.Println("1") - close(jobs) - pend.Wait() - - if len(data)%segmentsize != 0 { - count++ - } - restmp := make([]jobresult, leaflevelcount) - fmt.Println("2", leaflevelcount) - //Get computation results - for i := 0; i < leaflevelcount; i++ { - tres := <-results - restmp[tres.id] = tres - } - fmt.Println("3", height) - for i := 1; i < height; i++ { - jobs := make(chan jobparam, mprocessors) - for w := 1; w <= mprocessors; w++ { - pend.Add(1) - go processor(pend, 2, jobs, results) - } - var indexres = 0 - for j := 0; j < (1<>uint(i); j++ { - pend.Add(1) - jobs <- jobparam{n0: restmp[indexres].n, n1: restmp[indexres+1].n, data: restmp[indexres+1].data, height: 2, id: j} - indexres += 2 - } - close(jobs) - pend.Wait() - if i < height-1 { - for l := 0; l < (1<>uint(i); l++ { - tmpres := <-results - restmp[tmpres.id] = tmpres - } - } - } - rootLabel := make([]byte, 0) - var res jobresult - if leaflevelcount == 1 { - res = restmp[0] - } else { - res = <-results - } - if height > 0 { - rootLabel = res.n.label - } - hash := rootHash(uint64(count), rootLabel) - t := BTree{uint64(count), res.n, hash} - return &t -} - // returns a node and the left over data not used by it -func buildNode(data [][]byte, height int) (*node, [][]byte) { +func buildNode(data []byte, height int) (*node, []byte) { + if height == 0 || len(data) == 0 { + return nil, data + } + if height == 1 { + // leaf + return &node{label: data[0:segmentsize]}, data[segmentsize:] + } + n0, data := buildNode(data, height-1) + n1, data := buildNode(data, height-1) + + hash := makeHash(n0, n1) + return &node{label: hash, children: [2]*node{n0, n1}}, data +} + +func buildNode2(data [][]byte, height int) (*node, [][]byte) { if height == 0 || len(data) == 0 { return nil, data } @@ -338,8 +347,8 @@ func buildNode(data [][]byte, height int) (*node, [][]byte) { // leaf return &node{label: data[0]}, data[1:] } - n0, data := buildNode(data, height-1) - n1, data := buildNode(data, height-1) + n0, data := buildNode2(data, height-1) + n1, data := buildNode2(data, height-1) hash := makeHash(n0, n1) return &node{label: hash, children: [2]*node{n0, n1}}, data @@ -356,38 +365,80 @@ func splitData(data []byte, size int) [][]byte { if len(data)%size != 0 { blocks = append(blocks, data[len(blocks)*size:]) } + height := GetHeight(uint64(len(blocks))) + for i := len(blocks); i < (1 << uint(height)); i++ { + blocks = append(blocks, nil) + } + // return blocks } +type inclusionproofs struct { + proofs []inclusionproof + offset int + len int +} + +type inclusionproof struct { + proof [][]byte + offset int + len int + index int +} + +func (t *BTree) GetInclusionProofs(offset int, length int) (proofs inclusionproofs, err error) { + + if offset+length > t.chunklen { + + return proofs, errors.New(fmt.Sprintf("wrong offset+len %d :chunklen:%d", offset+length, t.chunklen)) + } + + n := (offset%segmentsize+length)/segmentsize + 1 + + proofs.proofs = make([]inclusionproof, n+1) + var index int = 0 + var segment = offset / segmentsize + for i := segment; i <= segment+n; i++ { + proofs.proofs[index], err = t.InclusionProof(i) + index++ + } + proofs.len = length + proofs.offset = offset + return proofs, nil +} + // Return a [][]byte needed to prove the gkf of the item at the passed index // The payload of the item at index is the first value in the proof -func (t *BTree) InclusionProof(index int) [][]byte { +func (t *BTree) InclusionProof(index int) (proof inclusionproof, err error) { if uint64(index) >= t.count { - panic("Invalid index: too large") + return proof, errors.New("Invalid index: too large") } if index < 0 { - panic("Invalid index: negative") + return proof, errors.New("Invalid index: negative") } h := GetHeight(t.count) - fmt.Println(h) - return proveNode(h, t.root, index) + proof.proof, err = proveNode(h, t.root, index) + proof.offset = index * segmentsize + proof.len = segmentsize + proof.index = index + return proof, err } -func proveNode(height int, n *node, index int) [][]byte { +func proveNode(height int, n *node, index int) ([][]byte, error) { if height == 1 { if index != 0 { - panic("Invalid index: non 0 for final node") + return nil, errors.New("Invalid index: non 0 for final node") } - return [][]byte{n.label} + return [][]byte{n.label}, nil } childIndex := index >> uint(height-2) nextIndex := index & (^(1 << uint(height-2))) - b := proveNode(height-1, n.children[childIndex], nextIndex) + b, _ := proveNode(height-1, n.children[childIndex], nextIndex) otherChildIndex := (childIndex + 1) % 2 if n.children[otherChildIndex] != nil { b = append(b, n.children[otherChildIndex].label) } - return b + return b, nil } // The Root of a merkle tree for a client that does not store the tree @@ -396,6 +447,18 @@ type Root struct { Base []byte } +func (r *Root) CheckProofs(h Hasher, proofs inclusionproofs) (bool, error) { + n := (proofs.offset%segmentsize+proofs.len)/segmentsize + 1 + + for i := 0; i < n; i++ { + ok, err := r.CheckProof(h, proofs.proofs[i].proof, proofs.proofs[i].index) + if (ok == false) || (err != nil) { + return ok, err + } + } + return true, nil +} + // Proves theof an element at the given index with the value thats the first entry in proof func (r *Root) CheckProof(h Hasher, proof [][]byte, index int) (bool, error) { hashFunc = h @@ -484,7 +547,8 @@ func (d *state) Reset() { // Write absorbs more data into the hash's state. func (d *state) Write(p []byte) (written int, err error) { - tree, r, count, err1 := BuildBMT(hashFunc, p, 32) + + tree, r, count, err1 := BuildBMT(hashFunc, p, true) d.btree = *tree d.root = *r diff --git a/swarm/storage/binarymerkle_test.go b/swarm/storage/binarymerkle_test.go index ec604db583..d78f0b0b02 100644 --- a/swarm/storage/binarymerkle_test.go +++ b/swarm/storage/binarymerkle_test.go @@ -3,84 +3,182 @@ package storage import ( "fmt" "log" + "math/rand" "testing" "time" "github.com/ethereum/go-ethereum/crypto/sha3" ) -//will test the hash (hash.Hash) interface -func TestBuildBMT2(t *testing.T) { +func TestBuildBMT(t *testing.T) { + for n := 0; n <= 4096; n += 1 { + fmt.Println("chunksize", n) + testBuildBMTprv(n, t) + } +} - // Grab some data to make the tree out of, and partition - data := make([]byte, 4096) - tdata := testDataReader(4096) +func testBuildBMTprv(n int, t *testing.T) { + + data := make([]byte, n) + tdata := testDataReader(n) tdata.Read(data) - var thashFunc = MakeHashFunc("BMTSHA3") - var h = thashFunc() - h.Reset() - h.Write(data) - var key = h.Sum(nil) - fmt.Println(key) + var tree *BTree + var r *Root + var count int + var err1 error + start := time.Now() + tree, r, count, err1 = BuildBMT(sha3.NewKeccak256, data, true) + elapsed := time.Since(start) + log.Printf("n=%d took %s", n, elapsed) + if err1 != nil { + fmt.Println(tree, r, count, err1) + return + } // for i := 0; i < count; i++ { - // p := tree.InclusionProof(i) + // p, err := tree.InclusionProof(i) + // if err != nil { + // fmt.Println("proof failed ", i, err.Error()) + // continue + // } + // ok, err := r.CheckProof(sha3.NewKeccak256, p.proof, i) // - // fmt.Println(p) - // - // ok := r.CheckProof(sha3.NewKeccak256, p, i) - // if !ok { + // if !ok || (err != nil) { // t.Errorf("proof %d failed", i) // } // } - // fmt.Println("done") -} -func TestBuildBMTStress(t *testing.T) { - - for i := 1; i < 4096; i++ { - BuildBMTwithGivenDataLen(i, t) - } -} - -func TestBuildBMT(t *testing.T) { - BuildBMTwithGivenDataLen(4096, t) -} - -func BuildBMTwithGivenDataLen(datalen int, t *testing.T) { - - // Grab some data to make the tree out of, and partition - fmt.Println("dl", datalen) - data := make([]byte, datalen) - tdata := testDataReader(datalen) - tdata.Read(data) - fmt.Println(data) - - start := time.Now() - - tree, r, count, err1 := BuildBMT(sha3.NewKeccak256, data, 32) - - elapsed := time.Since(start) - log.Printf("Binomial took %s", elapsed) - - if err1 != nil { - fmt.Println(err1) + offset := rand.Intn(n) + length := rand.Intn((n-offset+1)-1) + 1 + p, err := tree.GetInclusionProofs(offset, length) + if err != nil { + t.Errorf("proof %d failed %s", offset, err) return + } - fmt.Println(tree.Root(), count) + ok, err := r.CheckProofs(sha3.NewKeccak256, p) - for i := 0; i < count; i++ { - p := tree.InclusionProof(i) - - fmt.Println(p) - - ok, err := r.CheckProof(sha3.NewKeccak256, p, i) - - if !ok || (err != nil) { - t.Errorf("proof %d failed", i) - } + if !ok || (err != nil) { + t.Errorf("proof failed %s", err) + } else { + fmt.Println("proofs ok for offset", offset, "lenght", length, "chunksize", n) } + + // ok, err := r.CheckProof(sha3.NewKeccak256, p.proof, i) + // + // if !ok || (err != nil) { + // t.Errorf("proof %d failed", i) + // } + fmt.Println("done") } + +func benchmarkBuildBMT(n int, t *testing.B) { + //t.ReportAllocs() + tdata := testDataReader(n) + data := make([]byte, n) + tdata.Read(data) + + //reader := bytes.NewReader(data) + + var tree *BTree + var r *Root + var count int + var err1 error + // blocks := splitData(data, 32) + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + + tree, r, count, err1 = BuildBMT(sha3.NewKeccak256, data, false) + + if err1 != nil { + fmt.Println(err1, tree, r, count) + return + } + } +} + +func benchmarkSHA3(n int, t *testing.B) { + + data := make([]byte, n) + tdata := testDataReader(n) + tdata.Read(data) + hashFunc = sha3.NewKeccak256 + + t.ReportAllocs() + t.ResetTimer() + + h := hashFunc() + for i := 0; i < t.N; i++ { + + h.Reset() + h.Write(data) + //binary.Write(h, binary.LittleEndian, count) + h.Sum(nil) + + } + +} + +func BenchmarkBuildBMT_4k(t *testing.B) { benchmarkBuildBMT(4096, t) } +func BenchmarkBuildBMT_2k(t *testing.B) { benchmarkBuildBMT(4096/2, t) } +func BenchmarkBuildBMT_1k(t *testing.B) { benchmarkBuildBMT(4096/4, t) } +func BenchmarkBuildBMT_512b(t *testing.B) { benchmarkBuildBMT(4096/8, t) } +func BenchmarkBuildBMT_256b(t *testing.B) { benchmarkBuildBMT(4096/16, t) } +func BenchmarkBuildBMT_128b(t *testing.B) { benchmarkBuildBMT(4096/64, t) } + +func BenchmarkBuildSHA3_4k(t *testing.B) { benchmarkSHA3(4096, t) } +func BenchmarkBuildSHA3_2k(t *testing.B) { benchmarkSHA3(4096/2, t) } +func BenchmarkBuildSHA3_1k(t *testing.B) { benchmarkSHA3(4096/4, t) } +func BenchmarkBuildSHA3_512b(t *testing.B) { benchmarkSHA3(4096/8, t) } +func BenchmarkBuildSHA3_256b(t *testing.B) { benchmarkSHA3(4096/16, t) } + +func BenchmarkBuildNagiBinaryMerkle_4k(t *testing.B) { + n := 4096 + data := make([]byte, n) + tdata := testDataReader(n) + tdata.Read(data) + hashFunc = sha3.NewKeccak256 + + t.ReportAllocs() + t.ResetTimer() + + //h := hashFunc() + for i := 0; i < t.N; i++ { + + BinaryMerkle(data, sha3.NewKeccak256) + + } + +} + +//func BenchmarkBinaryMerkleTree(t *testing.B) { benchmarkBMT(4096, t) } + +// This implementation does not take advantage of any paralellisms and uses +// far more memory than necessary, but it is easy to see that it is correct. +// It can be used for generating test cases for optimized implementations. + +func BinaryMerkle(chunk []byte, hasher Hasher) []byte { + hash := hasher() + section := 2 * hash.Size() + l := len(chunk) + if l > section { + n := l / section + r := l - n*section + hash.Write(chunk[0:r]) + next := hash.Sum(nil) + for r < l { + hash.Reset() + hash.Write(chunk[r : r+section]) + next = hash.Sum(next) + r += section + } + return BinaryMerkle(next, hasher) + } else { + hash.Write(chunk) + return hash.Sum(nil) + } +}