From c3bb14e3520163f4a45ec0c927ca3430a44ffbd5 Mon Sep 17 00:00:00 2001
From: zsfelfoldi <zsfelfoldi@gmail.com>
Date: Fri, 29 May 2015 03:13:37 +0200
Subject: [PATCH 1/2] bzz.Api.Upload determines mime types fixed absolute path
 conversion

---
 bzz/api.go | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/bzz/api.go b/bzz/api.go
index 8322252519..2df66f6dba 100644
--- a/bzz/api.go
+++ b/bzz/api.go
@@ -7,6 +7,7 @@ import (
 	"io"
 	"net"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -110,9 +111,12 @@ func (self *Api) Download(bzzpath, localpath string) (string, error) {
 // Upload replicates a local directory as a manifest file and uploads it
 // using dpa store
 // TODO: localpath should point to a manifest
-func (self *Api) Upload(localpath string) (string, error) {
+func (self *Api) Upload(lpath string) (string, error) {
 	var files []string
-	localpath = common.ExpandHomePath(localpath)
+	localpath, err1 := filepath.Abs(lpath)
+	if err1 != nil {
+		return "", err1
+	}
 	start := len(localpath)
 	if (start > 0) && (localpath[start-1] != os.PathSeparator) {
 		start++
@@ -125,7 +129,7 @@ func (self *Api) Upload(localpath string) (string, error) {
 				return fmt.Errorf("Path is too short")
 			}
 			if path[:len(localpath)] != localpath {
-				return fmt.Errorf("Path prefix does not match localpath")
+				return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath)
 			}
 			files = append(files, path)
 		}
@@ -138,6 +142,7 @@ func (self *Api) Upload(localpath string) (string, error) {
 	cnt := len(files)
 	hashes := make([]Key, cnt)
 	errors := make([]error, cnt)
+	ctypes := make([]string, cnt)
 	wg := &sync.WaitGroup{}
 
 	for i, path := range files {
@@ -149,6 +154,15 @@ func (self *Api) Upload(localpath string) (string, error) {
 				sr := io.NewSectionReader(f, 0, stat.Size())
 				hashes[i], err = self.dpa.Store(sr, wg)
 			}
+			if err == nil {
+				cmd := exec.Command("file", "--mime-type", "-b", path)
+				var out bytes.Buffer
+				cmd.Stdout = &out
+				err = cmd.Run()
+				if err == nil {
+					ctypes[i] = strings.TrimSuffix(out.String(), "\n")
+				}
+			}
 			errors[i] = err
 			wg.Done()
 		}(i, path)
@@ -169,7 +183,7 @@ func (self *Api) Upload(localpath string) (string, error) {
 		if i == cnt-1 {
 			sc = "]}"
 		}
-		buffer.WriteString(fmt.Sprintf(`{"hash":"%064x","path":"%s","contentType":"text/plain"}%s`, hashes[i], path[start:], sc))
+		buffer.WriteString(fmt.Sprintf(`{"hash":"%064x","path":"%s","contentType":"%s"}%s`, hashes[i], path[start:], ctypes[i], sc))
 	}
 
 	manifest := buffer.Bytes()

From ab75309ffaff14fadacfe3804f948b3e55aa4692 Mon Sep 17 00:00:00 2001
From: zsfelfoldi <zsfelfoldi@gmail.com>
Date: Fri, 29 May 2015 03:33:18 +0200
Subject: [PATCH 2/2] fixed too many open files bug, limited parallel
 processing of files

---
 bzz/api.go | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/bzz/api.go b/bzz/api.go
index 2df66f6dba..05fefa7796 100644
--- a/bzz/api.go
+++ b/bzz/api.go
@@ -108,12 +108,14 @@ func (self *Api) Download(bzzpath, localpath string) (string, error) {
 	return "", nil
 }
 
+const maxParallelFiles = 5
+
 // Upload replicates a local directory as a manifest file and uploads it
 // using dpa store
 // TODO: localpath should point to a manifest
 func (self *Api) Upload(lpath string) (string, error) {
 	var files []string
-	localpath, err1 := filepath.Abs(lpath)
+	localpath, err1 := filepath.Abs(filepath.Clean(lpath))
 	if err1 != nil {
 		return "", err1
 	}
@@ -143,16 +145,22 @@ func (self *Api) Upload(lpath string) (string, error) {
 	hashes := make([]Key, cnt)
 	errors := make([]error, cnt)
 	ctypes := make([]string, cnt)
-	wg := &sync.WaitGroup{}
+	done := make(chan bool, maxParallelFiles)
+	dcnt := 0
 
 	for i, path := range files {
-		wg.Add(1)
-		go func(i int, path string) {
+		if i >= dcnt+maxParallelFiles {
+			<-done
+			dcnt++
+		}
+		go func(i int, path string, done chan bool) {
 			f, err := os.Open(path)
 			if err == nil {
 				stat, _ := f.Stat()
 				sr := io.NewSectionReader(f, 0, stat.Size())
+				wg := &sync.WaitGroup{}
 				hashes[i], err = self.dpa.Store(sr, wg)
+				wg.Wait()
 			}
 			if err == nil {
 				cmd := exec.Command("file", "--mime-type", "-b", path)
@@ -164,10 +172,13 @@ func (self *Api) Upload(lpath string) (string, error) {
 				}
 			}
 			errors[i] = err
-			wg.Done()
-		}(i, path)
+			done <- true
+		}(i, path, done)
+	}
+	for dcnt < cnt {
+		<-done
+		dcnt++
 	}
-	wg.Wait()
 
 	var buffer bytes.Buffer
 	buffer.WriteString(`{"entries":[`)
@@ -188,6 +199,7 @@ func (self *Api) Upload(lpath string) (string, error) {
 
 	manifest := buffer.Bytes()
 	sr := io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest)))
+	wg := &sync.WaitGroup{}
 	key, err2 := self.dpa.Store(sr, wg)
 	wg.Wait()
 	return fmt.Sprintf("%064x", key), err2