From a762134519d8cdd4898bd8ff4ddf3e38de9b44bd Mon Sep 17 00:00:00 2001 From: zsfelfoldi Date: Sat, 30 May 2015 09:22:38 +0200 Subject: [PATCH] implemented manifestTrie.getEntry, removed old manifest.go --- bzz/api.go | 66 +++++---------------------------------- bzz/manifest.go | 47 ---------------------------- bzz/manifest_trie.go | 73 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 79 insertions(+), 107 deletions(-) delete mode 100644 bzz/manifest.go diff --git a/bzz/api.go b/bzz/api.go index 8c8a247433..18eb533b8f 100644 --- a/bzz/api.go +++ b/bzz/api.go @@ -2,7 +2,6 @@ package bzz import ( "bytes" - "encoding/json" "fmt" "io" "net" @@ -286,67 +285,18 @@ func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, sta return } - // retrieve content following path along manifests - var pos int - for { - dpaLogger.Debugf("Swarm: manifest lookup key: '%064x'.", key) - // retrieve manifest via DPA - manifestReader := self.dpa.Retrieve(key) - // TODO check size for oversized manifests - manifestData := make([]byte, manifestReader.Size()) - var size int - size, err = manifestReader.Read(manifestData) - if int64(size) < manifestReader.Size() { - dpaLogger.Debugf("Swarm: Manifest for '%s' not found.", uri) - if err == nil { - err = fmt.Errorf("Manifest retrieval cut short: %v < %v", size, manifestReader.Size()) - } - return - } + trie, err := loadManifestTrie(self.dpa, key) + if err != nil { + return + } - dpaLogger.Debugf("Swarm: Manifest for '%s' retrieved", uri) - man := manifest{} - err = json.Unmarshal(manifestData, &man) - if err != nil { - err = fmt.Errorf("Manifest for '%s' is malformed: %v", uri, err) - dpaLogger.Debugf("Swarm: %v", err) - return - } - - dpaLogger.Debugf("Swarm: Manifest for '%s' has %d entries. Retrieving entry for '%s'", uri, len(man.Entries), path) - - // retrieve entry that matches path from manifest entries - var entry *manifestEntry - entry, pos = man.getEntry(path) - if entry == nil { - err = fmt.Errorf("Content for '%s' not found.", uri) - return - } - - // check hash of entry - if !hashMatcher.MatchString(entry.Hash) { - err = fmt.Errorf("Incorrect hash '%064x' for '%s'", entry.Hash, uri) - return - } + entry, _ := trie.getEntry(self.dpa, path) + if entry != nil { key = common.Hex2Bytes(entry.Hash) status = entry.Status - - // get mime type of entry mimeType = entry.ContentType - if mimeType == "" { - mimeType = manifestType - } - - // if path matched on non-manifest content type, then retrieve reader - // and return - if mimeType != manifestType { - dpaLogger.Debugf("Swarm: content lookup key: '%064x' (%v)", key, mimeType) - reader = self.dpa.Retrieve(key) - return - } - - // otherwise continue along the path with manifest resolution - path = path[pos:] + dpaLogger.Debugf("Swarm: content lookup key: '%064x' (%v)", key, mimeType) + reader = self.dpa.Retrieve(key) } return } diff --git a/bzz/manifest.go b/bzz/manifest.go deleted file mode 100644 index 47b9d99ebb..0000000000 --- a/bzz/manifest.go +++ /dev/null @@ -1,47 +0,0 @@ -package bzz - -import ( - // "fmt" - "regexp" -) - -const ( - manifestType = "application/bzz-manifest+json" -) - -var ( - leadingSlashes = regexp.MustCompile("^/+") -) - -type manifest struct { - Entries []*manifestEntry `json:"entries"` -} - -type manifestEntry struct { - Path string `json:"path"` - Hash string `json:"hash"` - ContentType string `json:"contentType"` - Status int `json:"status"` -} - -func (self *manifest) getEntry(path string) (entry *manifestEntry, pos int) { - for _, entry = range self.Entries { - entryPath := leadingSlashes.ReplaceAllString(entry.Path, "") - pos = len(entryPath) - if len(path) >= pos && path[:pos] == entryPath { - var n int - if len(path) > pos { - chopped := leadingSlashes.ReplaceAllString(path[pos:], "") - n = len(path) - pos - len(chopped) - } - if n > 0 || pos == 0 || path[pos-1] == '/' { - pos += n - dpaLogger.Debugf("Swarm: '%s' matches '%s'.", path, entry.Path) - return - } - } - } - entry = nil - dpaLogger.Debugf("Path '%s' on manifest not found.", path) - return -} diff --git a/bzz/manifest_trie.go b/bzz/manifest_trie.go index 524b40c6ee..6b164337aa 100644 --- a/bzz/manifest_trie.go +++ b/bzz/manifest_trie.go @@ -6,6 +6,12 @@ import ( "fmt" "io" "sync" + + "github.com/ethereum/go-ethereum/common" +) + +const ( + manifestType = "application/bzz-manifest+json" ) type manifestTrie struct { @@ -21,6 +27,7 @@ type manifestTrieEntry struct { Path string `json:"path"` Hash string `json:"hash"` // for manifest content type, empty until subtrie is evaluated ContentType string `json:"contentType"` + Status int `json:"status"` subtrie *manifestTrie } @@ -121,12 +128,12 @@ func (self *manifestTrie) deleteEntry(path string) { } b := byte(path[0]) - if (self.entries[b] != nil) && (self.entries[b].Path == path) { + entry := self.entries[b] + if (entry != nil) && (entry.Path == path) { self.entries[b] = nil return } - entry := self.entries[b] epl := len(entry.Path) if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) { entry.subtrie.deleteEntry(path[epl:]) @@ -176,3 +183,65 @@ func (self *manifestTrie) recalcAndStore(dpa *DPA) error { self.hash = key return err2 } + +func (self *manifestTrie) findPrefixOf(dpa *DPA, path string) (entry *manifestTrieEntry, pos int) { + if len(path) == 0 { + return self.entries[256], 0 + } + + b := byte(path[0]) + entry = self.entries[b] + epl := len(entry.Path) + if (len(path) >= epl) && (path[:epl] == entry.Path) { + if entry.ContentType == manifestType { + if entry.subtrie == nil { + hash := common.Hex2Bytes(entry.Hash) + var err error + entry.subtrie, err = loadManifestTrie(dpa, hash) + if err != nil { + return nil, 0 + } + entry.Hash = "" // might not match, should be recalculated + } + entry, pos = entry.subtrie.findPrefixOf(dpa, path[epl:]) + if entry != nil { + pos += epl + } + } else { + pos = epl + } + } else { + entry = nil + } + return +} + +func (self *manifestTrie) getEntryNLS(dpa *DPA, path string) (entry *manifestTrieEntry, pos int) { + entry, pos = self.findPrefixOf(dpa, path) + if entry != nil { + for (pos < len(path)) && (path[pos] == '/') { + pos++ + } + if (pos > 0) && (path[pos-1] != '/') { + return nil, 0 + } + } + return +} + +func (self *manifestTrie) getEntry(dpa *DPA, path string) (entry *manifestTrieEntry, pos int) { + var slash string + for { + entry, pos = self.getEntryNLS(dpa, slash+path) + if pos < len(slash) { + dpaLogger.Debugf("Path '%s' on manifest not found.", path) + return nil, 0 + } + if entry != nil { + pos -= len(slash) + dpaLogger.Debugf("Swarm: '%s' matches '%s'.", path, entry.Path) + return + } + slash = slash + "/" + } +}