forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-21 20:26:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix division by zero in split

Browse source 
- update chunk pretty printer
- add comments
This commit is contained in:
zelig 2015-02-01 19:35:45 +01:00
parent 99190409dd
commit 4d3af2ca33
1 changed files with 410 additions and 400 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

810
bzz/chunker.go
View file

@ -1,400 +1,410 @@
/* /*
The distributed storage implemented in this package requires fix sized chunks of content The distributed storage implemented in this package requires fix sized chunks of content
Chunker is the interface to a component that is responsible for disassembling and assembling larger data. Chunker is the interface to a component that is responsible for disassembling and assembling larger data.
TreeChunker implements a Chunker based on a tree structure defined as follows: TreeChunker implements a Chunker based on a tree structure defined as follows:
1 each node in the tree including the root and other branching nodes are stored as a chunk. 1 each node in the tree including the root and other branching nodes are stored as a chunk.
2 branching nodes encode data contents that includes the size of the dataslice covered by its entire subtree under the node as well as the hash keys of all its children : 2 branching nodes encode data contents that includes the size of the dataslice covered by its entire subtree under the node as well as the hash keys of all its children :
data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1} data_{i} := size(subtree_{i}) || key_{j} || key_{j+1} .... || key_{j+n-1}
3 Leaf nodes encode an actual subslice of the input data. 3 Leaf nodes encode an actual subslice of the input data.
4 if data size is not more than maximum chunksize, the data is stored in a single chunk 4 if data size is not more than maximum chunksize, the data is stored in a single chunk
key = sha256(int64(size) + data) key = sha256(int64(size) + data)
2 if data size is more than chunksize*Branches^l, but no more than chunksize* 2 if data size is more than chunksize*Branches^l, but no more than chunksize*
Branches^(l+1), the data vector is split into slices of chunksize* Branches^(l+1), the data vector is split into slices of chunksize*
Branches^l length (except the last one). Branches^l length (except the last one).
key = sha256(int64(size) + key(slice0) + key(slice1) + ...) key = sha256(int64(size) + key(slice0) + key(slice1) + ...)
*/ */
package bzz package bzz
import ( import (
"crypto" "crypto"
"encoding/binary" "encoding/binary"
"fmt" "fmt"
"io" "io"
"sync" "sync"
"time" "time"
) )
const ( const (
hasherfunc crypto.Hash = crypto.SHA256 // http://golang.org/pkg/hash/#Hash hasherfunc crypto.Hash = crypto.SHA256 // http://golang.org/pkg/hash/#Hash
branches int64 = 4 branches int64 = 4
) )
var ( var (
// hashSize int64 = hasherfunc.New().Size() // hasher knows about its own length in bytes // hashSize int64 = hasherfunc.New().Size() // hasher knows about its own length in bytes
// chunksize int64 = branches * hashSize // chunk is defined as this // chunksize int64 = branches * hashSize // chunk is defined as this
joinTimeout = 120 * time.Second joinTimeout = 120 * time.Second
splitTimeout = 120 * time.Second splitTimeout = 120 * time.Second
) )
type Key []byte type Key []byte
/* /*
Chunker is the interface to a component that is responsible for disassembling and assembling larger data and indended to be the dependency of a DPA storage system with fixed maximum chunksize. Chunker is the interface to a component that is responsible for disassembling and assembling larger data and indended to be the dependency of a DPA storage system with fixed maximum chunksize.
It relies on the underlying chunking model. It relies on the underlying chunking model.
When calling Split, the caller provides a channel (chan *Chunk) on which it receives chunks to store. The DPA delegates to storage layers (implementing ChunkStore interface). NewChunkstore(DB) is a convenience wrapper with which all DBs (conforming to DB interface) can serve as ChunkStores. See chunkStore.go When calling Split, the caller provides a channel (chan *Chunk) on which it receives chunks to store. The DPA delegates to storage layers (implementing ChunkStore interface). NewChunkstore(DB) is a convenience wrapper with which all DBs (conforming to DB interface) can serve as ChunkStores. See chunkStore.go
After getting notified that all the data has been split (the error channel is closed), the caller can safely read or save the root key. Optionally it times out if not all chunks get stored or not the entire stream of data has been processed. By inspecting the errc channel the caller can check if any explicit errors (typically IO read/write failures) occured during splitting. After getting notified that all the data has been split (the error channel is closed), the caller can safely read or save the root key. Optionally it times out if not all chunks get stored or not the entire stream of data has been processed. By inspecting the errc channel the caller can check if any explicit errors (typically IO read/write failures) occured during splitting.
When calling Join with a root key, the caller gets returned a lazy reader. The caller again provides a channel and receives an error channel. The chunk channel is the one on which the caller receives placeholder chunks with missing data. The DPA is supposed to forward this to the chunk stores and notify the chunker if the data has been delivered (i.e. retrieved from memory cache, disk-persisted db or cloud based swarm delivery). The chunker then puts these together and notifies the DPA if data has been assembled by a closed error channel. Once the DPA finds the data has been joined, it is free to deliver it back to swarm in full (if the original request was via the bzz protocol) or save and serve if it it was a local client request. When calling Join with a root key, the caller gets returned a lazy reader. The caller again provides a channel and receives an error channel. The chunk channel is the one on which the caller receives placeholder chunks with missing data. The DPA is supposed to forward this to the chunk stores and notify the chunker if the data has been delivered (i.e. retrieved from memory cache, disk-persisted db or cloud based swarm delivery). The chunker then puts these together and notifies the DPA if data has been assembled by a closed error channel. Once the DPA finds the data has been joined, it is free to deliver it back to swarm in full (if the original request was via the bzz protocol) or save and serve if it it was a local client request.
*/ */
type Chunker interface { type Chunker interface {
/* /*
When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Key), the root hash of the entire content will fill this once processing finishes. When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Key), the root hash of the entire content will fill this once processing finishes.
New chunks to store are coming to caller via the chunk storage channel, which the caller provides. New chunks to store are coming to caller via the chunk storage channel, which the caller provides.
The caller gets returned an error channel, if an error is encountered during splitting, it is fed to errC error channel. The caller gets returned an error channel, if an error is encountered during splitting, it is fed to errC error channel.
A closed error signals process completion at which point the key can be considered final if there were no errors. A closed error signals process completion at which point the key can be considered final if there were no errors.
*/ */
Split(key Key, data SectionReader, chunkC chan *Chunk) chan error Split(key Key, data SectionReader, chunkC chan *Chunk) chan error
/* /*
Join reconstructs original content based on a root key. Join reconstructs original content based on a root key.
When joining, the caller gets returned a LazySectionReader and an error channel. When joining, the caller gets returned a LazySectionReader and an error channel.
New chunks to retrieve are coming to caller via the Chunk channel, which the caller provides. New chunks to retrieve are coming to caller via the Chunk channel, which the caller provides.
If an error is encountered during joining, it is fed to errC error channel. If an error is encountered during joining, it is fed to errC error channel.
A closed error signals process completion at which point the data can be considered final and fully reconstructed if there were no errors. A closed error signals process completion at which point the data can be considered final and fully reconstructed if there were no errors.
The LazySectionReader provides on-demand fetching of content including chunks. The LazySectionReader provides on-demand fetching of content including chunks.
Lifecycle of the reader can be modified with SetTimeout() Lifecycle of the reader can be modified with SetTimeout()
*/ */
Join(key Key, chunkC chan *Chunk) (LazySectionReader, chan error) Join(key Key, chunkC chan *Chunk) (LazySectionReader, chan error)
} }
/* /*
Tree chunker is a concrete implementation of data chunking. Tree chunker is a concrete implementation of data chunking.
This chunker works in a simple way, it builds a tree out of the document so that each node either represents a chunk of real data or a chunk of data representing an branching non-leaf node of the tree. In particular each such non-leaf chunk will represent is a concatenation of the hash of its respective children. This scheme simultaneously guarantees data integrity as well as self addressing. Abstract nodes are transparent since their represented size component is strictly greater than their maximum data size, since they encode a subtree. This chunker works in a simple way, it builds a tree out of the document so that each node either represents a chunk of real data or a chunk of data representing an branching non-leaf node of the tree. In particular each such non-leaf chunk will represent is a concatenation of the hash of its respective children. This scheme simultaneously guarantees data integrity as well as self addressing. Abstract nodes are transparent since their represented size component is strictly greater than their maximum data size, since they encode a subtree.
If all is well it is possible to implement this by simply composing readers so that no extra allocation or buffering is necessary for the data splitting and joining. This means that in principle there can be direct IO between : memory, file system, network socket (bzz peers storage request is read from the socket ). In practice there may be need for several stages of internal buffering. If all is well it is possible to implement this by simply composing readers so that no extra allocation or buffering is necessary for the data splitting and joining. This means that in principle there can be direct IO between : memory, file system, network socket (bzz peers storage request is read from the socket ). In practice there may be need for several stages of internal buffering.
Unfortunately the hashing itself does use extra copies and allocation though since it does need it. Unfortunately the hashing itself does use extra copies and allocation though since it does need it.
*/ */
type TreeChunker struct { type TreeChunker struct {
Branches int64 Branches int64
HashFunc crypto.Hash HashFunc crypto.Hash
JoinTimeout time.Duration JoinTimeout time.Duration
SplitTimeout time.Duration SplitTimeout time.Duration
// calculated // calculated
hashSize int64 // self.HashFunc.New().Size() hashSize int64 // self.HashFunc.New().Size()
chunkSize int64 // hashSize* Branches chunkSize int64 // hashSize* Branches
} }
func (self *TreeChunker) Init() { func (self *TreeChunker) Init() {
if self.HashFunc == 0 { if self.HashFunc == 0 {
self.HashFunc = hasherfunc self.HashFunc = hasherfunc
} }
if self.Branches == 0 { if self.Branches == 0 {
self.Branches = branches self.Branches = branches
} }
if self.JoinTimeout == 0 { if self.JoinTimeout == 0 {
self.JoinTimeout = joinTimeout self.JoinTimeout = joinTimeout
} }
if self.SplitTimeout == 0 { if self.SplitTimeout == 0 {
self.SplitTimeout = splitTimeout self.SplitTimeout = splitTimeout
} }
self.hashSize = int64(self.HashFunc.New().Size()) self.hashSize = int64(self.HashFunc.New().Size())
self.chunkSize = self.hashSize * self.Branches self.chunkSize = self.hashSize * self.Branches
dpaLogger.Debugf("Chunker initialised: branches: %v, hashsize: %v, chunksize: %v, join timeout: %v , split timeout: %v", self.Branches, self.hashSize, self.chunkSize, self.JoinTimeout, self.SplitTimeout) dpaLogger.Debugf("Chunker initialised: branches: %v, hashsize: %v, chunksize: %v, join timeout: %v , split timeout: %v", self.Branches, self.hashSize, self.chunkSize, self.JoinTimeout, self.SplitTimeout)
} }
func (self *TreeChunker) HashSize() int64 { func (self *TreeChunker) HashSize() int64 {
return self.hashSize return self.hashSize
} }
type Chunk struct { // Chunk serves also serves as a request object passed to ChunkStores
Data SectionReader // nil if request, to be supplied by dpa // in case it is a retrieval request, Data is nil and Size is 0
Size int64 // size of the data covered by the subtree encoded in this chunk // Note that Size is not the size of the data chunk, which is Data.Size() see SectionReader
// not the size of data, which is Data.Size() see SectionReader // but the size of the subtree encoded in the chunk
// 0 if request, to be supplied by dpa // 0 if request, to be supplied by the dpa
Key Key // always type Chunk struct {
C chan bool // to signal data delivery by the dpa Data SectionReader // nil if request, to be supplied by dpa
wg sync.WaitGroup Size int64 // size of the data covered by the subtree encoded in this chunk
} Key Key // always
C chan bool // to signal data delivery by the dpa
func (self *Chunk) String() string { update bool //
var size int64 }
var slice []byte
if self.Data != nil { // String() for pretty printing
size = self.Data.Size() func (self *Chunk) String() string {
slice = make([]byte, size) var size int64
self.Data.ReadAt(slice, 0) var slice []byte
} var n int
return fmt.Sprintf("Key: [%x..] TreeSize: %v Chunksize: %v Data: %x\n", self.Key[:4], self.Size, size, slice) var err error
} if self.Data != nil {
size = 32 // we are printing 32 bytes of the data
// The treeChunkers own Hash hashes together slice = make([]byte, size)
// - the size (of the subtree encoded in the Chunk) n, err = self.Data.ReadAt(slice, 0)
// - the Chunk, ie. the contents read from the input reader if err != nil && err != io.EOF {
func (self *TreeChunker) Hash(size int64, input SectionReader) []byte { slice = []byte(fmt.Sprintf("ERROR: %v", err))
hasher := self.HashFunc.New() }
binary.Write(hasher, binary.LittleEndian, size) }
io.Copy(hasher, input) // it uses WriteTo if available, ChunkReader implements io.WriterTo return fmt.Sprintf("Key: [%x..] TreeSize: %v Chunksize: %v Data: %x\n", self.Key[:4], self.Size, size, slice[:n])
return hasher.Sum(nil) }
}
// The treeChunkers own Hash hashes together
func (self *TreeChunker) Split(key Key, data SectionReader, chunkC chan *Chunk) (errC chan error) { // - the size (of the subtree encoded in the Chunk)
wg := &sync.WaitGroup{} // - the Chunk, ie. the contents read from the input reader
errC = make(chan error) func (self *TreeChunker) Hash(size int64, input SectionReader) []byte {
rerrC := make(chan error) hasher := self.HashFunc.New()
timeout := time.After(splitTimeout) binary.Write(hasher, binary.LittleEndian, size)
if key == nil { io.Copy(hasher, input) // it uses WriteTo if available, ChunkReader implements io.WriterTo
dpaLogger.Debugf("please allocate byte slice for root key") return hasher.Sum(nil)
return }
}
wg.Add(1) func (self *TreeChunker) Split(key Key, data SectionReader, chunkC chan *Chunk) (errC chan error) {
go func() { wg := &sync.WaitGroup{}
errC = make(chan error)
depth := 0 rerrC := make(chan error)
treeSize := self.chunkSize timeout := time.After(splitTimeout)
size := data.Size() if key == nil {
// takes lowest depth such that chunksize*HashCount^(depth+1) > size dpaLogger.Debugf("please allocate byte slice for root key")
// power series, will find the order of magnitude of the data size in base hashCount or numbers of levels of branching in the resulting tree. return
}
for ; treeSize < size; treeSize *= self.Branches { wg.Add(1)
depth++ go func() {
}
depth := 0
dpaLogger.Debugf("split request received for data (%v bytes, depth: %v)", size, depth) treeSize := self.chunkSize
size := data.Size()
//launch actual recursive function passing the workgroup // takes lowest depth such that chunksize*HashCount^(depth+1) > size
self.split(depth, treeSize/self.Branches, key, data, chunkC, rerrC, wg) // power series, will find the order of magnitude of the data size in base hashCount or numbers of levels of branching in the resulting tree.
}()
for ; treeSize < size; treeSize *= self.Branches {
// closes internal error channel if all subprocesses in the workgroup finished depth++
go func() { }
wg.Wait()
close(rerrC) dpaLogger.Debugf("split request received for data (%v bytes, depth: %v)", size, depth)
}() //launch actual recursive function passing the workgroup
self.split(depth, treeSize/self.Branches, key, data, chunkC, rerrC, wg)
// waiting for request to end with wg finishing, error, or timeout }()
go func() {
select { // closes internal error channel if all subprocesses in the workgroup finished
case err := <-rerrC: go func() {
if err != nil { wg.Wait()
errC <- err close(rerrC)
} // otherwise splitting is complete
case <-timeout: }()
errC <- fmt.Errorf("split time out")
} // waiting for request to end with wg finishing, error, or timeout
close(errC) go func() {
}() select {
case err := <-rerrC:
return if err != nil {
} errC <- err
} // otherwise splitting is complete
func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionReader, chunkC chan *Chunk, errc chan error, parentWg *sync.WaitGroup) { case <-timeout:
errC <- fmt.Errorf("split time out")
defer parentWg.Done() }
close(errC)
size := data.Size() }()
var newChunk *Chunk
var hash Key return
dpaLogger.Debugf("depth: %v, max subtree size: %v, data size: %v", depth, treeSize, size) }
if depth == 0 { func (self *TreeChunker) split(depth int, treeSize int64, key Key, data SectionReader, chunkC chan *Chunk, errc chan error, parentWg *sync.WaitGroup) {
// leaf nodes -> content chunks
hash = self.Hash(size, data) defer parentWg.Done()
dpaLogger.Debugf("content chunk: max subtree size: %v, data size: %v", treeSize, size)
newChunk = &Chunk{ size := data.Size()
Key: hash, var newChunk *Chunk
Data: data, var hash Key
Size: size, dpaLogger.Debugf("depth: %v, max subtree size: %v, data size: %v", depth, treeSize, size)
}
} else { for depth > 0 && size < treeSize {
for size < treeSize { treeSize /= self.Branches
treeSize /= self.Branches depth--
depth-- }
}
// intermediate chunk containing child nodes hashes if depth == 0 {
branches := int64((size-1)/treeSize) + 1 // leaf nodes -> content chunks
dpaLogger.Debugf("intermediate node: setting branches: %v, depth: %v, max subtree size: %v, data size: %v", branches, depth, treeSize, size) hash = self.Hash(size, data)
dpaLogger.Debugf("content chunk: max subtree size: %v, data size: %v", treeSize, size)
var chunk []byte = make([]byte, branches*self.hashSize) newChunk = &Chunk{
var pos, i int64 Key: hash,
Data: data,
childrenWg := &sync.WaitGroup{} Size: size,
var secSize int64 }
for i < branches { } else {
// the last item can have shorter data // intermediate chunk containing child nodes hashes
if size-pos < treeSize { branches := int64((size-1)/treeSize) + 1
secSize = size - pos dpaLogger.Debugf("intermediate node: setting branches: %v, depth: %v, max subtree size: %v, data size: %v", branches, depth, treeSize, size)
} else {
secSize = treeSize var chunk []byte = make([]byte, branches*self.hashSize)
} var pos, i int64
// take the section of the data corresponding encoded in the subTree
subTreeData := NewChunkReader(data, pos, secSize) childrenWg := &sync.WaitGroup{}
// the hash of that data var secSize int64
subTreeKey := chunk[i*self.hashSize : (i+1)*self.hashSize] for i < branches {
// the last item can have shorter data
childrenWg.Add(1) if size-pos < treeSize {
go self.split(depth-1, treeSize/self.Branches, subTreeKey, subTreeData, chunkC, errc, childrenWg) secSize = size - pos
} else {
i++ secSize = treeSize
pos += treeSize }
} // take the section of the data corresponding encoded in the subTree
// wait for all the children to complete calculating their hashes and copying them onto sections of the chunk subTreeData := NewChunkReader(data, pos, secSize)
childrenWg.Wait() // the hash of that data
// now we got the hashes in the chunk, then hash the chunk subTreeKey := chunk[i*self.hashSize : (i+1)*self.hashSize]
chunkReader := NewChunkReaderFromBytes(chunk) // bytes.Reader almost implements SectionReader
hash = self.Hash(size, chunkReader) childrenWg.Add(1)
newChunk = &Chunk{ go self.split(depth-1, treeSize/self.Branches, subTreeKey, subTreeData, chunkC, errc, childrenWg)
Key: hash,
Data: chunkReader, i++
Size: size, pos += treeSize
} }
} // wait for all the children to complete calculating their hashes and copying them onto sections of the chunk
// send off new chunk to storage childrenWg.Wait()
if chunkC != nil { // now we got the hashes in the chunk, then hash the chunk
chunkC <- newChunk chunkReader := NewChunkReaderFromBytes(chunk) // bytes.Reader almost implements SectionReader
} hash = self.Hash(size, chunkReader)
// report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)x newChunk = &Chunk{
copy(key, hash) Key: hash,
Data: chunkReader,
} Size: size,
}
func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) (data LazySectionReader, errC chan error) { }
// initialise return parameters // send off new chunk to storage
errC = make(chan error) if chunkC != nil {
// timer to time out the operation (needed within so as to avoid process leakage) chunkC <- newChunk
timeout := time.After(joinTimeout) }
wg := &sync.WaitGroup{} // report hash of this chunk one level up (keys corresponds to the proper subslice of the parent chunk)x
// initialise internal error channel copy(key, hash)
rerrC := make(chan error)
quitC := make(chan bool) }
// launch lazy recursive call on root chunk func (self *TreeChunker) Join(key Key, chunkC chan *Chunk) (data LazySectionReader, errC chan error) {
notReadyReader := &NotReadyReader{} // initialise return parameters
reader := &EmbeddedReader{ errC = make(chan error)
LazySectionReader: &lazyReader{notReadyReader}, // timer to time out the operation (needed within so as to avoid process leakage)
} timeout := time.After(joinTimeout)
fun := self.join(0, 0, key, chunkC, rerrC, wg, quitC) wg := &sync.WaitGroup{}
data = reader // initialise internal error channel
rerrC := make(chan error)
// processing is triggered by reads on the LazySectionReader quitC := make(chan bool)
wg.Add(1)
notReadyReader.r = reader // launch lazy recursive call on root chunk
notReadyReader.initF = func() { notReadyReader := &NotReadyReader{}
reader.lock.Lock() reader := &EmbeddedReader{
if fun != nil { LazySectionReader: &lazyReader{notReadyReader},
reader.LazySectionReader = fun() // replace the reader while caller is waiting }
wg.Done() // just to have one in waitgroup until reader funcs are called fun := self.join(0, 0, key, chunkC, rerrC, wg, quitC)
fun = nil // to be only called once data = reader
}
reader.lock.Unlock() // processing is triggered by reads on the LazySectionReader
} wg.Add(1)
notReadyReader.r = reader
// waits for all the processes to finish and signals by closing internal rerrc notReadyReader.initF = func() {
go func() { reader.lock.Lock()
wg.Wait() if fun != nil {
close(rerrC) reader.LazySectionReader = fun() // replace the reader while caller is waiting
}() wg.Done() // just to have one in waitgroup until reader funcs are called
fun = nil // to be only called once
go func() { }
select { reader.lock.Unlock()
case err := <-rerrC: }
if err != nil {
errC <- err // waits for all the processes to finish and signals by closing internal rerrc
} // otherwise channel is closed, data joining complete go func() {
case <-timeout: wg.Wait()
errC <- fmt.Errorf("join time out") close(rerrC)
close(quitC) }()
}
// this will indicate to the caller that processing is finished (with or without error) go func() {
close(errC) select {
}() case err := <-rerrC:
if err != nil {
return errC <- err
} } // otherwise channel is closed, data joining complete
case <-timeout:
func (self *TreeChunker) join(depth int, treeSize int64, key Key, chunkC chan *Chunk, errC chan error, wg *sync.WaitGroup, quitC chan bool) (readerF func() LazySectionReader) { errC <- fmt.Errorf("join time out")
wg.Add(1) close(quitC)
readerF = func() (r LazySectionReader) { }
defer wg.Done() // this will indicate to the caller that processing is finished (with or without error)
chunk := &Chunk{ close(errC)
Key: key, }()
C: make(chan bool, 1), // close channel to signal data delivery
} return
chunkC <- chunk // submit retrieval request, someone should be listening on the other side (or we will time out globally) }
// waiting for the chunk retrieval func (self *TreeChunker) join(depth int, treeSize int64, key Key, chunkC chan *Chunk, errC chan error, wg *sync.WaitGroup, quitC chan bool) (readerF func() LazySectionReader) {
select { wg.Add(1)
case <-quitC: readerF = func() (r LazySectionReader) {
// this is how we control process leakage (quitC is closed once join is finished (after timeout)) defer wg.Done()
return chunk := &Chunk{
case <-chunk.C: // bells are ringing, data have been delivered Key: key,
} C: make(chan bool, 1), // close channel to signal data delivery
}
// calculate depth and max treeSize chunkC <- chunk // submit retrieval request, someone should be listening on the other side (or we will time out globally)
var depth int
var treeSize int64 = self.hashSize // waiting for the chunk retrieval
for ; treeSize*self.Branches < chunk.Size; treeSize *= self.Branches { select {
depth++ case <-quitC:
} // this is how we control process leakage (quitC is closed once join is finished (after timeout))
return
if depth == 0 { case <-chunk.C: // bells are ringing, data have been delivered
return LazyReader(chunk.Data) // simply give back the chunks reader for content chunks }
}
// calculate depth and max treeSize
// find appropriate block level var depth int
for chunk.Size < treeSize { var treeSize int64 = self.hashSize
treeSize /= self.Branches for ; treeSize*self.Branches < chunk.Size; treeSize *= self.Branches {
depth-- depth++
} }
// boooo
// intermediate chunk, chunk containing hashes of child nodes if depth == 0 {
var pos, i, secSize int64 return LazyReader(chunk.Data) // simply give back the chunks reader for content chunks
var childKey Key }
var readerF func() (r LazySectionReader)
var readerFs [](func() (r LazySectionReader)) // find appropriate block level
branches := int64((chunk.Size-1)/treeSize) + 1 for chunk.Size < treeSize {
dpaLogger.DebugDetailf("tree node - size %v, chunk size: %v, subtreeSize %v, branches %v", chunk.Size, chunk.Data.Size(), treeSize, branches) treeSize /= self.Branches
depth--
// iterate through the chunk containing the keys of children }
// create lazy init functions that give back readers // boooo
for i < branches { // intermediate chunk, chunk containing hashes of child nodes
if chunk.Size-pos < treeSize { var pos, i, secSize int64
secSize = chunk.Size - pos var childKey Key
dpaLogger.DebugDetailf("tree node section %v: size %v", i, secSize) var readerF func() (r LazySectionReader)
} else { var readerFs [](func() (r LazySectionReader))
secSize = treeSize branches := int64((chunk.Size-1)/treeSize) + 1
} dpaLogger.DebugDetailf("tree node - size %v, chunk size: %v, subtreeSize %v, branches %v", chunk.Size, chunk.Data.Size(), treeSize, branches)
// create partial Chunk in order to send a retrieval request
childKey = make([]byte, self.hashSize) // preallocate hashSize long slice for key // iterate through the chunk containing the keys of children
// read the Hash of the subtree from the relevant section of the Chunk into the allocated byte slice in subtree.Key // create lazy init functions that give back readers
if _, err := chunk.Data.ReadAt(childKey, i*self.hashSize); err != nil { for i < branches {
dpaLogger.DebugDetailf("Read error: %v", err) if chunk.Size-pos < treeSize {
errC <- err secSize = chunk.Size - pos
break dpaLogger.DebugDetailf("tree node section %v: size %v", i, secSize)
} } else {
// call lazy reader function recursively on the subtree secSize = treeSize
readerF = self.join(depth-1, treeSize/self.Branches, childKey, chunkC, errC, wg, quitC) }
readerFs = append(readerFs, readerF) // create partial Chunk in order to send a retrieval request
i++ childKey = make([]byte, self.hashSize) // preallocate hashSize long slice for key
pos += treeSize // read the Hash of the subtree from the relevant section of the Chunk into the allocated byte slice in subtree.Key
} if _, err := chunk.Data.ReadAt(childKey, i*self.hashSize); err != nil {
// new reader created on demand: dpaLogger.DebugDetailf("Read error: %v", err)
r = &LazyChunkReader{ errC <- err
readerFs: readerFs, break
sections: make([]LazySectionReader, branches), }
size: chunk.Size, // call lazy reader function recursively on the subtree
treeSize: treeSize, readerF = self.join(depth-1, treeSize/self.Branches, childKey, chunkC, errC, wg, quitC)
} readerFs = append(readerFs, readerF)
return i++
} pos += treeSize
return }
} // new reader created on demand:
r = &LazyChunkReader{
readerFs: readerFs,
sections: make([]LazySectionReader, branches),
size: chunk.Size,
treeSize: treeSize,
}
return
}
return
}