merge in menifestTrie changes to api

bzz/api.go

@@ -2,7 +2,6 @@ package bzz
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -123,6 +122,31 @@ func (self *Api) Put(content, contentType string) (string, error) {
 	return fmt.Sprintf("%064x", key), nil
 }
 
+func (self *Api) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
+	root := common.Hex2Bytes(rootHash)
+	trie, err := loadManifestTrie(self.dpa, root)
+	if err != nil {
+		return
+	}
+
+	if contentHash != "" {
+		entry := &manifestTrieEntry{
+			Path:        path,
+			Hash:        contentHash,
+			ContentType: contentType,
+		}
+		trie.addEntry(entry)
+	} else {
+		trie.deleteEntry(path)
+	}
+
+	err = trie.recalcAndStore()
+	if err != nil {
+		return
+	}
+	return fmt.Sprintf("%064x", trie.hash), nil
+}
+
 // Download replicates the manifest path structure on the local filesystem
 // under localpath
 func (self *Api) Download(bzzpath, localpath string) (string, error) {
@@ -135,7 +159,7 @@ const maxParallelFiles = 5
 // using dpa store
 // TODO: localpath should point to a manifest
 func (self *Api) Upload(lpath string) (string, error) {
-	var files []string
+	var list []*manifestTrieEntry
 	localpath, err1 := filepath.Abs(filepath.Clean(lpath))
 	if err1 != nil {
 		return "", err1
@@ -154,7 +178,10 @@ func (self *Api) Upload(lpath string) (string, error) {
 			if path[:len(localpath)] != localpath {
 				return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath)
 			}
-			files = append(files, path)
+			entry := &manifestTrieEntry{
+				Path: path,
+			}
+			list = append(list, entry)
 		}
 		return err
 	})
@@ -162,68 +189,64 @@ func (self *Api) Upload(lpath string) (string, error) {
 		return "", err
 	}
 
-	cnt := len(files)
-	hashes := make([]Key, cnt)
+	cnt := len(list)
 	errors := make([]error, cnt)
-	ctypes := make([]string, cnt)
 	done := make(chan bool, maxParallelFiles)
 	dcnt := 0
 
-	for i, path := range files {
+	for i, entry := range list {
 		if i >= dcnt+maxParallelFiles {
 			<-done
 			dcnt++
 		}
-		go func(i int, path string, done chan bool) {
-			f, err := os.Open(path)
+		go func(i int, entry *manifestTrieEntry, done chan bool) {
+			f, err := os.Open(entry.Path)
 			if err == nil {
 				stat, _ := f.Stat()
 				sr := io.NewSectionReader(f, 0, stat.Size())
 				wg := &sync.WaitGroup{}
-				hashes[i], err = self.dpa.Store(sr, wg)
+				var hash Key
+				hash, err = self.dpa.Store(sr, wg)
+				if hash != nil {
+					list[i].Hash = fmt.Sprintf("%064x", hash)
+				}
 				wg.Wait()
 			}
 			if err == nil {
-				cmd := exec.Command("file", "--mime-type", "-b", path)
+				cmd := exec.Command("file", "--mime-type", "-b", entry.Path)
 				var out bytes.Buffer
 				cmd.Stdout = &out
 				err = cmd.Run()
 				if err == nil {
-					ctypes[i] = strings.TrimSuffix(out.String(), "\n")
+					list[i].ContentType = strings.TrimSuffix(out.String(), "\n")
 				}
 			}
 			errors[i] = err
 			done <- true
-		}(i, path, done)
+		}(i, entry, done)
 	}
 	for dcnt < cnt {
 		<-done
 		dcnt++
 	}
 
-	var buffer bytes.Buffer
-	buffer.WriteString(`{"entries":[`)
-	sc := ","
-	if err != nil {
-		return "", err
+	trie := &manifestTrie{
+		dpa: self.dpa,
 	}
-
-	for i, path := range files {
+	for i, entry := range list {
 		if errors[i] != nil {
 			return "", errors[i]
 		}
-		if i == cnt-1 {
-			sc = "]}"
-		}
-		buffer.WriteString(fmt.Sprintf(`{"hash":"%064x","path":"%s","contentType":"%s"}%s`, hashes[i], path[start:], ctypes[i], sc))
+		entry.Path = entry.Path[start:]
+		trie.addEntry(entry)
 	}
 
-	manifest := buffer.Bytes()
-	sr := io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest)))
-	wg := &sync.WaitGroup{}
-	key, err2 := self.dpa.Store(sr, wg)
-	wg.Wait()
-	return fmt.Sprintf("%064x", key), err2
+	err2 := trie.recalcAndStore()
+	var hs string
+	if err2 == nil {
+		hs = fmt.Sprintf("%064x", trie.hash)
+	}
+	return hs, err2
 }
 
 func (self *Api) Register(sender common.Address, hash common.Hash, domain string) (err error) {
@@ -289,67 +312,18 @@ func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, sta
 		return
 	}
 
-	// retrieve content following path along manifests
-	var pos int
-	for {
-		dpaLogger.Debugf("Swarm: manifest lookup key: '%064x'.", key)
-		// retrieve manifest via DPA
-		manifestReader := self.dpa.Retrieve(key)
-		// TODO check size for oversized manifests
-		manifestData := make([]byte, manifestReader.Size())
-		var size int
-		size, err = manifestReader.Read(manifestData)
-		if int64(size) < manifestReader.Size() {
-			dpaLogger.Debugf("Swarm: Manifest for '%s' not found (uri: '%s')", path[:pos], uri)
-			if err == nil {
-				err = fmt.Errorf("Manifest retrieval cut short: %v &lt; %v", size, manifestReader.Size())
-			}
-			return
-		}
+	trie, err := loadManifestTrie(self.dpa, key)
+	if err != nil {
+		return
+	}
 
-		dpaLogger.Debugf("Swarm: Manifest for '%s' retrieved", path[:pos])
-		man := manifest{}
-		err = json.Unmarshal(manifestData, &man)
-		if err != nil {
-			err = fmt.Errorf("Manifest for '%s' is malformed: %v", path[:pos], err)
-			dpaLogger.Debugf("Swarm: %v", err)
-			return
-		}
-
-		dpaLogger.Debugf("Swarm: Manifest for '%s' has %d entries. Match path '%s'", path[:pos], len(man.Entries), path[pos:])
-
-		// retrieve entry that matches path from manifest entries
-		entry, hop := man.getEntry(path[pos:])
-
-		if entry == nil {
-			err = fmt.Errorf("Path '%s' not found on manifest '%s'", path[:pos], path[pos:])
-			return
-		}
-
-		// check hash of entry
-		if !hashMatcher.MatchString(entry.Hash) {
-			err = fmt.Errorf("Incorrect hash '%064x' for path '%s' on manifest for '%s')", entry.Hash, path[pos:], path[:pos])
-			return
-		}
+	entry, _ := trie.getEntry(path)
+	if entry != nil {
 		key = common.Hex2Bytes(entry.Hash)
 		status = entry.Status
-
-		// get mime type of entry
 		mimeType = entry.ContentType
-		if mimeType == "" {
-			mimeType = manifestType
-		}
-
-		// if path matched on non-manifest content type, then retrieve reader
-		// and return
-		if mimeType != manifestType {
-			dpaLogger.Debugf("Swarm: content lookup key: '%064x' (%v)", key, mimeType)
-			reader = self.dpa.Retrieve(key)
-			return
-		}
-
-		// otherwise continue along the path with manifest resolution
-		pos += hop
+		dpaLogger.Debugf("Swarm: content lookup key: '%064x' (%v)", key, mimeType)
+		reader = self.dpa.Retrieve(key)
 	}
 	return
 }

bzz/manifest.go

@@ -1,64 +0,0 @@
-package bzz
-
-import (
-	// "fmt"
-	"regexp"
-)
-
-const (
-	manifestType = "application/bzz-manifest+json"
-)
-
-var (
-	leadingSlashes = regexp.MustCompile("^/+")
-)
-
-type manifest struct {
-	Entries []*manifestEntry `json:"entries"`
-}
-
-type manifestEntry struct {
-	Path        string `json:"path"`
-	Hash        string `json:"hash"`
-	ContentType string `json:"contentType"`
-	Status      int    `json:"status"`
-}
-
-// path must not have any leading slashes
-func (self *manifest) getEntry(path string) (matchingEntry *manifestEntry, matchlength int) {
-	var pos, depth, maxdepth int
-	var entry *manifestEntry
-	// path := leadingSlashes.ReplaceAllString(fullpath, "")
-	// iterate over entries matching paths to the target
-	// due to redundant slashes, it is NOT the longest match but the match with
-	// the highest depth is chosen
-	// this gives thse matches in case of trailing slashes:
-	// "a/" -> "a/" not "a" and "a" matches "a" not "a/" if both exist
-	// "a" never matches "a/" but "a/" matches a
-	for _, entry = range self.Entries {
-		entryPath := leadingSlashes.ReplaceAllString(entry.Path, "")
-		pos = len(entryPath)
-		depth = len(slashes.Split(entryPath, -1))
-		if len(path) >= pos && path[:pos] == entryPath && (depth > maxdepth || depth == maxdepth && pos > matchlength) {
-			var hop int
-			// hop and chop leading hashes of the continuation
-			if len(path) > pos {
-				chopped := leadingSlashes.ReplaceAllString(path[pos:], "")
-				hop = len(path) - pos - len(chopped)
-			}
-			// check if pos actually ends a subpath "ab" matches on "" not on "a"
-			if hop > 0 || pos == len(path) || pos == 0 || path[pos-1] == '/' {
-				matchlength = pos + hop
-				maxdepth = depth
-				matchingEntry = entry
-			}
-		}
-	}
-	if matchingEntry != nil {
-		dpaLogger.Debugf("Swarm: '%s' matches '%s'.", path, matchingEntry.Path)
-	} else {
-		dpaLogger.Debugf("Path '%s' not found on manifest ", path)
-	}
-
-	return
-}

bzz/manifest_trie.go

@@ -0,0 +1,261 @@
+package bzz
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+const (
+	manifestType = "application/bzz-manifest+json"
+)
+
+type manifestTrie struct {
+	dpa     *DPA
+	entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry
+	hash    Key                     // if hash != nil, it is stored
+}
+
+type manifestJSON struct {
+	Entries []*manifestTrieEntry `json:"entries"`
+}
+
+type manifestTrieEntry struct {
+	Path        string `json:"path"`
+	Hash        string `json:"hash"` // for manifest content type, empty until subtrie is evaluated
+	ContentType string `json:"contentType"`
+	Status      int    `json:"status"`
+	subtrie     *manifestTrie
+}
+
+func loadManifestTrie(dpa *DPA, hash Key) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
+
+	dpaLogger.Debugf("Swarm: manifest lookup key: '%064x'.", hash)
+	// retrieve manifest via DPA
+	manifestReader := dpa.Retrieve(hash)
+	// TODO check size for oversized manifests
+	manifestData := make([]byte, manifestReader.Size())
+	var size int
+	size, err = manifestReader.Read(manifestData)
+	if int64(size) < manifestReader.Size() {
+		dpaLogger.Debugf("Swarm: Manifest %064x not found.", hash)
+		if err == nil {
+			err = fmt.Errorf("Manifest retrieval cut short: %v &lt; %v", size, manifestReader.Size())
+		}
+		return
+	}
+
+	dpaLogger.Debugf("Swarm: Manifest %064x retrieved", hash)
+	man := manifestJSON{}
+	err = json.Unmarshal(manifestData, &man)
+	if err != nil {
+		err = fmt.Errorf("Manifest %064x is malformed: %v", hash, err)
+		dpaLogger.Debugf("Swarm: %v", err)
+		return
+	}
+
+	dpaLogger.Debugf("Swarm: Manifest %064x has %d entries.", hash, len(man.Entries))
+
+	trie = &manifestTrie{
+		dpa: dpa,
+	}
+	for _, entry := range man.Entries {
+		trie.addEntry(entry)
+	}
+	return
+}
+
+func (self *manifestTrie) addEntry(entry *manifestTrieEntry) {
+	self.hash = nil // trie modified, hash needs to be re-calculated on demand
+
+	if len(entry.Path) == 0 {
+		self.entries[256] = entry
+		return
+	}
+
+	b := byte(entry.Path[0])
+	if (self.entries[b] == nil) || (self.entries[b].Path == entry.Path) {
+		self.entries[b] = entry
+		return
+	}
+
+	oldentry := self.entries[b]
+	cpl := 0
+	for (len(entry.Path) > cpl) && (len(oldentry.Path) > cpl) && (entry.Path[cpl] == oldentry.Path[cpl]) {
+		cpl++
+	}
+
+	if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) {
+		if self.loadSubTrie(oldentry) != nil {
+			return
+		}
+		entry.Path = entry.Path[cpl:]
+		oldentry.subtrie.addEntry(entry)
+		oldentry.Hash = ""
+		return
+	}
+
+	commonPrefix := entry.Path[:cpl]
+
+	subtrie := &manifestTrie{
+		dpa: self.dpa,
+	}
+	entry.Path = entry.Path[cpl:]
+	oldentry.Path = oldentry.Path[cpl:]
+	subtrie.addEntry(entry)
+	subtrie.addEntry(oldentry)
+
+	self.entries[b] = &manifestTrieEntry{
+		Path:        commonPrefix,
+		Hash:        "",
+		ContentType: manifestType,
+		subtrie:     subtrie,
+	}
+}
+
+func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
+	for _, e := range self.entries {
+		if e != nil {
+			cnt++
+			entry = e
+		}
+	}
+	return
+}
+
+func (self *manifestTrie) deleteEntry(path string) {
+	self.hash = nil // trie modified, hash needs to be re-calculated on demand
+
+	if len(path) == 0 {
+		self.entries[256] = nil
+		return
+	}
+
+	b := byte(path[0])
+	entry := self.entries[b]
+	if (entry != nil) && (entry.Path == path) {
+		self.entries[b] = nil
+		return
+	}
+
+	epl := len(entry.Path)
+	if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
+		if self.loadSubTrie(entry) != nil {
+			return
+		}
+		entry.subtrie.deleteEntry(path[epl:])
+		entry.Hash = ""
+		// remove subtree if it has less than 2 elements
+		cnt, lastentry := entry.subtrie.getCountLast()
+		if cnt < 2 {
+			if lastentry != nil {
+				lastentry.Path = entry.Path + lastentry.Path
+			}
+			self.entries[b] = lastentry
+		}
+	}
+}
+
+func (self *manifestTrie) recalcAndStore() error {
+	if self.hash != nil {
+		return nil
+	}
+
+	var buffer bytes.Buffer
+	buffer.WriteString(`{"entries":[`)
+
+	list := &manifestJSON{}
+	for _, entry := range self.entries {
+		if entry != nil {
+			if entry.Hash == "" { // TODO: paralellize
+				err := entry.subtrie.recalcAndStore()
+				if err != nil {
+					return err
+				}
+				entry.Hash = fmt.Sprintf("%064x", entry.subtrie.hash)
+			}
+			list.Entries = append(list.Entries, entry)
+		}
+	}
+
+	manifest, err := json.Marshal(list)
+	if err != nil {
+		return err
+	}
+
+	sr := io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest)))
+	wg := &sync.WaitGroup{}
+	key, err2 := self.dpa.Store(sr, wg)
+	wg.Wait()
+	self.hash = key
+	return err2
+}
+
+func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry) (err error) {
+	if entry.subtrie == nil {
+		hash := common.Hex2Bytes(entry.Hash)
+		entry.subtrie, err = loadManifestTrie(self.dpa, hash)
+		entry.Hash = "" // might not match, should be recalculated
+	}
+	return
+}
+
+func (self *manifestTrie) findPrefixOf(path string) (entry *manifestTrieEntry, pos int) {
+	if len(path) == 0 {
+		return self.entries[256], 0
+	}
+
+	b := byte(path[0])
+	entry = self.entries[b]
+	epl := len(entry.Path)
+	if (len(path) >= epl) && (path[:epl] == entry.Path) {
+		if entry.ContentType == manifestType {
+			if self.loadSubTrie(entry) != nil {
+				return nil, 0
+			}
+			entry, pos = entry.subtrie.findPrefixOf(path[epl:])
+			if entry != nil {
+				pos += epl
+			}
+		} else {
+			pos = epl
+		}
+	} else {
+		entry = nil
+	}
+	return
+}
+
+func (self *manifestTrie) getEntryNLS(path string) (entry *manifestTrieEntry, pos int) {
+	entry, pos = self.findPrefixOf(path)
+	if entry != nil {
+		for (pos < len(path)) && (path[pos] == '/') {
+			pos++
+		}
+		if (pos > 0) && (path[pos-1] != '/') {
+			return nil, 0
+		}
+	}
+	return
+}
+
+func (self *manifestTrie) getEntry(path string) (entry *manifestTrieEntry, pos int) {
+	var slash string
+	for {
+		entry, pos = self.getEntryNLS(slash + path)
+		if pos < len(slash) {
+			dpaLogger.Debugf("Path '%s' on manifest not found.", path)
+			return nil, 0
+		}
+		if entry != nil {
+			pos -= len(slash)
+			dpaLogger.Debugf("Swarm: '%s' matches '%s'.", path, entry.Path)
+			return
+		}
+		slash = slash + "/"
+	}
+}