From ca6b22ee7c6c45122163c860b92bdebdc6896c82 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 30 Mar 2018 17:19:33 +0200 Subject: [PATCH 01/10] swarm/...: Integrate encryption into swarm API --- swarm/api/api.go | 18 +- swarm/api/api_test.go | 9 +- swarm/api/filesystem.go | 4 +- swarm/api/filesystem_test.go | 33 +- swarm/api/http/server.go | 23 +- swarm/api/http/server_test.go | 6 +- swarm/api/manifest.go | 13 +- swarm/api/storage.go | 4 +- swarm/api/storage_test.go | 10 +- swarm/fuse/swarmfs_test.go | 912 ++++++++++++++++++---------------- swarm/storage/dpa.go | 4 + 11 files changed, 551 insertions(+), 485 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index be09809613..57b6300cf7 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -39,7 +39,9 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) -var hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}") + +// TODO: this is bad, it should not be hardcoded how long is a hash +var hashMatcher = regexp.MustCompile("^([0-9A-Fa-f]{64})([0-9A-Fa-f]{64})?") type ErrResourceReturn struct { key string @@ -230,9 +232,9 @@ func NewApi(dpa *storage.DPA, dns Resolver, resourceHandler *storage.ResourceHan } // to be used only in TEST -func (self *Api) Upload(uploadDir, index string) (hash string, err error) { +func (self *Api) Upload(uploadDir, index string, toEncrypt bool) (hash string, err error) { fs := NewFileSystem(self) - hash, err = fs.Upload(uploadDir, index) + hash, err = fs.Upload(uploadDir, index, toEncrypt) return hash, err } @@ -241,9 +243,9 @@ func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader { return self.dpa.Retrieve(key) } -func (self *Api) Store(data io.Reader, size int64) (key storage.Key, wait func(), err error) { +func (self *Api) Store(data io.Reader, size int64, toEncrypt bool) (key storage.Key, wait func(), err error) { log.Debug("api.store", "size", size) - return self.dpa.Store(data, size, false) + return self.dpa.Store(data, size, toEncrypt) } type ErrResolve error @@ -283,17 +285,17 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { } // Put provides singleton manifest creation on top of dpa store -func (self *Api) Put(content, contentType string) (k storage.Key, wait func(), err error) { +func (self *Api) Put(content, contentType string, toEncrypt bool) (k storage.Key, wait func(), err error) { apiPutCount.Inc(1) r := strings.NewReader(content) - key, waitContent, err := self.dpa.Store(r, int64(len(content)), false) + key, waitContent, err := self.dpa.Store(r, int64(len(content)), toEncrypt) if err != nil { apiPutFail.Inc(1) return nil, nil, err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) r = strings.NewReader(manifest) - key, waitManifest, err := self.dpa.Store(r, int64(len(manifest)), false) + key, waitManifest, err := self.dpa.Store(r, int64(len(manifest)), toEncrypt) if err != nil { apiPutFail.Inc(1) return nil, nil, err diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index abbc8c0e96..7499c9d553 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) -func testApi(t *testing.T, f func(*Api)) { +func testApi(t *testing.T, f func(*Api, bool)) { datadir, err := ioutil.TempDir("", "bzz-test") if err != nil { t.Fatalf("unable to create temp dir: %v", err) @@ -43,7 +43,8 @@ func testApi(t *testing.T, f func(*Api)) { return } api := NewApi(dpa, nil, nil) - f(api) + f(api, false) + f(api, true) } type testResponse struct { @@ -106,11 +107,11 @@ func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse { } func TestApiPut(t *testing.T) { - testApi(t, func(api *Api) { + testApi(t, func(api *Api, toEncrypt bool) { content := "hello" exp := expResponse(content, "text/plain", 0) // exp := expResponse([]byte(content), "text/plain", 0) - key, wait, err := api.Put(content, exp.MimeType) + key, wait, err := api.Put(content, exp.MimeType, toEncrypt) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index 8de4f4ee3e..e816a17c92 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -47,7 +47,7 @@ func NewFileSystem(api *Api) *FileSystem { // TODO: localpath should point to a manifest // // DEPRECATED: Use the HTTP API instead -func (self *FileSystem) Upload(lpath, index string) (string, error) { +func (self *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error) { var list []*manifestTrieEntry localpath, err := filepath.Abs(filepath.Clean(lpath)) if err != nil { @@ -114,7 +114,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) { stat, _ := f.Stat() var hash storage.Key var wait func() - hash, wait, err = self.api.dpa.Store(f, stat.Size(), false) + hash, wait, err = self.api.dpa.Store(f, stat.Size(), toEncrypt) if hash != nil { list[i].Hash = hash.Hex() } diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index 6f1594e991..a73ed667c7 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -29,9 +29,9 @@ import ( var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test") -func testFileSystem(t *testing.T, f func(*FileSystem)) { - testApi(t, func(api *Api) { - f(NewFileSystem(api)) +func testFileSystem(t *testing.T, f func(*FileSystem, bool)) { + testApi(t, func(api *Api, toEncrypt bool) { + f(NewFileSystem(api), toEncrypt) }) } @@ -46,9 +46,9 @@ func readPath(t *testing.T, parts ...string) string { } func TestApiDirUpload0(t *testing.T) { - testFileSystem(t, func(fs *FileSystem) { + testFileSystem(t, func(fs *FileSystem, toEncrypt bool) { api := fs.api - bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "") + bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "", toEncrypt) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -74,20 +74,21 @@ func TestApiDirUpload0(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - newbzzhash, err := fs.Upload(downloadDir, "") + newbzzhash, err := fs.Upload(downloadDir, "", toEncrypt) if err != nil { t.Fatalf("unexpected error: %v", err) } - if bzzhash != newbzzhash { + // TODO: currently the hash is not deterministic in the encrypted case + if !toEncrypt && bzzhash != newbzzhash { t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash) } }) } func TestApiDirUploadModify(t *testing.T) { - testFileSystem(t, func(fs *FileSystem) { + testFileSystem(t, func(fs *FileSystem, toEncrypt bool) { api := fs.api - bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "") + bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "", toEncrypt) if err != nil { t.Errorf("unexpected error: %v", err) return @@ -104,7 +105,7 @@ func TestApiDirUploadModify(t *testing.T) { t.Errorf("unexpected error: %v", err) return } - hash, wait, err := api.Store(bytes.NewReader(index), int64(len(index))) + hash, wait, err := api.Store(bytes.NewReader(index), int64(len(index)), toEncrypt) wait() if err != nil { t.Errorf("unexpected error: %v", err) @@ -144,9 +145,9 @@ func TestApiDirUploadModify(t *testing.T) { } func TestApiDirUploadWithRootFile(t *testing.T) { - testFileSystem(t, func(fs *FileSystem) { + testFileSystem(t, func(fs *FileSystem, toEncrypt bool) { api := fs.api - bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "index.html") + bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "index.html", toEncrypt) if err != nil { t.Errorf("unexpected error: %v", err) return @@ -160,9 +161,9 @@ func TestApiDirUploadWithRootFile(t *testing.T) { } func TestApiFileUpload(t *testing.T) { - testFileSystem(t, func(fs *FileSystem) { + testFileSystem(t, func(fs *FileSystem, toEncrypt bool) { api := fs.api - bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "") + bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "", toEncrypt) if err != nil { t.Errorf("unexpected error: %v", err) return @@ -176,9 +177,9 @@ func TestApiFileUpload(t *testing.T) { } func TestApiFileUploadWithRootFile(t *testing.T) { - testFileSystem(t, func(fs *FileSystem) { + testFileSystem(t, func(fs *FileSystem, toEncrypt bool) { api := fs.api - bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "index.html") + bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "index.html", toEncrypt) if err != nil { t.Errorf("unexpected error: %v", err) return diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 6636f4c6a5..6b7e44437e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -124,19 +124,30 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { log.Debug("handle.post.raw", "ruid", r.ruid) postRawCount.Inc(1) + + toEncrypt := false + if r.uri.Addr == "encrypt" { + toEncrypt = true + } + if r.uri.Path != "" { postRawFail.Inc(1) Respond(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) return } + if r.uri.Addr != "" && r.uri.Addr != "encrypt" { + postRawFail.Inc(1) + Respond(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest) + return + } + if r.Header.Get("Content-Length") == "" { postRawFail.Inc(1) Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest) return } - - key, _, err := s.api.Store(r.Body, r.ContentLength) + key, _, err := s.api.Store(r.Body, r.ContentLength, toEncrypt) if err != nil { postRawFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) @@ -176,7 +187,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { } log.Debug("resolved key", "ruid", r.ruid, "key", key) } else { - key, err = s.api.NewManifest() + key, err = s.api.NewManifest(false) if err != nil { postFilesFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) @@ -365,7 +376,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { Respond(w, r, err2.Error(), code) return } - m, err := s.api.NewResourceManifest(r.uri.Addr) + m, err := s.api.NewResourceManifest(r.uri.Addr, false) if err != nil { Respond(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError) return @@ -840,14 +851,18 @@ func (s *Server) ServeHTTP(rw http.ResponseWriter, r *http.Request) { req.uri = uri log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri", req.uri) + log.Debug("parsed request path", "uri.Addr", req.uri.Addr, "uri.path", req.uri.Path, "uri.Scheme", req.uri.Scheme) switch r.Method { case "POST": if uri.Raw() || uri.DeprecatedRaw() { + log.Debug("handlePostRaw") s.HandlePostRaw(w, req) } else if uri.Resource() { + log.Debug("handlePostResource") s.HandlePostResource(w, req) } else { + log.Debug("handlePostFiles") s.HandlePostFiles(w, req) } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 292da70087..b94847f82d 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -218,7 +218,11 @@ func TestBzzResource(t *testing.T) { } func TestBzzGetPath(t *testing.T) { + // testBzzGetPath(false, t) + testBzzGetPath(true, t) +} +func testBzzGetPath(encrypted bool, t *testing.T) { var err error testmanifest := []string{ @@ -246,7 +250,7 @@ func TestBzzGetPath(t *testing.T) { for i, mf := range testmanifest { reader[i] = bytes.NewReader([]byte(mf)) var wait func() - key[i], wait, err = srv.Dpa.Store(reader[i], int64(len(mf)), false) + key[i], wait, err = srv.Dpa.Store(reader[i], int64(len(mf)), encrypted) if err != nil { t.Fatal(err) } diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index aaf0035d62..6047066a64 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -59,20 +59,20 @@ type ManifestList struct { } // NewManifest creates and stores a new, empty manifest -func (a *Api) NewManifest() (storage.Key, error) { +func (a *Api) NewManifest(toEncrypt bool) (storage.Key, error) { var manifest Manifest data, err := json.Marshal(&manifest) if err != nil { return nil, err } - key, wait, err := a.Store(bytes.NewReader(data), int64(len(data))) + key, wait, err := a.Store(bytes.NewReader(data), int64(len(data)), toEncrypt) wait() return key, err } // Manifest hack for supporting Mutable Resource Updates from the bzz: scheme // see swarm/api/api.go:Api.Get() for more information -func (a *Api) NewResourceManifest(resourceKey string) (storage.Key, error) { +func (a *Api) NewResourceManifest(resourceKey string, toEncrypt bool) (storage.Key, error) { var manifest Manifest entry := ManifestEntry{ Hash: resourceKey, @@ -83,7 +83,7 @@ func (a *Api) NewResourceManifest(resourceKey string) (storage.Key, error) { if err != nil { return nil, err } - key, _, err := a.Store(bytes.NewReader(data), int64(len(data))) + key, _, err := a.Store(bytes.NewReader(data), int64(len(data)), toEncrypt) return key, err } @@ -104,7 +104,10 @@ func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWrit // AddEntry stores the given data and adds the resulting key to the manifest func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) { - key, _, err := m.api.Store(data, e.Size) + + toEncrypt := (len(m.trie.hash) > m.trie.dpa.HashSize()) + + key, _, err := m.api.Store(data, e.Size, toEncrypt) if err != nil { return nil, err } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index a2ba70c7ac..464e00877a 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -45,8 +45,8 @@ func NewStorage(api *Api) *Storage { // its content type // // DEPRECATED: Use the HTTP API instead -func (self *Storage) Put(content, contentType string) (storage.Key, func(), error) { - return self.api.Put(content, contentType) +func (self *Storage) Put(content, contentType string, toEncrypt bool) (storage.Key, func(), error) { + return self.api.Put(content, contentType, toEncrypt) } // Get retrieves the content from bzzpath and reads the response in full diff --git a/swarm/api/storage_test.go b/swarm/api/storage_test.go index bcbf53ee37..9f1e17506b 100644 --- a/swarm/api/storage_test.go +++ b/swarm/api/storage_test.go @@ -20,18 +20,18 @@ import ( "testing" ) -func testStorage(t *testing.T, f func(*Storage)) { - testApi(t, func(api *Api) { - f(NewStorage(api)) +func testStorage(t *testing.T, f func(*Storage, bool)) { + testApi(t, func(api *Api, toEncrypt bool) { + f(NewStorage(api), toEncrypt) }) } func TestStoragePutGet(t *testing.T) { - testStorage(t, func(api *Storage) { + testStorage(t, func(api *Storage, toEncrypt bool) { content := "hello" exp := expResponse(content, "text/plain", 0) // exp := expResponse([]byte(content), "text/plain", 0) - bzzkey, wait, err := api.Put(content, exp.MimeType) + bzzkey, wait, err := api.Put(content, exp.MimeType, toEncrypt) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 0186749522..b385a72587 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -38,7 +38,7 @@ type fileInfo struct { contents []byte } -func createTestFilesAndUploadToSwarm(t *testing.T, api *api.Api, files map[string]fileInfo, uploadDir string) string { +func createTestFilesAndUploadToSwarm(t *testing.T, api *api.Api, files map[string]fileInfo, uploadDir string, toEncrypt bool) string { os.RemoveAll(uploadDir) for fname, finfo := range files { @@ -62,9 +62,9 @@ func createTestFilesAndUploadToSwarm(t *testing.T, api *api.Api, files map[strin fd.Close() } - bzzhash, err := api.Upload(uploadDir, "") + bzzhash, err := api.Upload(uploadDir, "", toEncrypt) if err != nil { - t.Fatalf("Error uploading directory %v: %v", uploadDir, err) + t.Fatalf("Error uploading directory %v: %vm encryption: %v", uploadDir, err, toEncrypt) } return bzzhash @@ -171,7 +171,7 @@ func checkFile(t *testing.T, testMountDir, fname string, contents []byte) { } } -func getRandomBtes(size int) []byte { +func getRandomBytes(size int) []byte { contents := make([]byte, size) rand.Read(contents) return contents @@ -198,78 +198,86 @@ func (ta *testAPI) mountListAndUnmount(t *testing.T) { testUploadDir, _ := ioutil.TempDir(os.TempDir(), "fuse-source") testMountDir, _ := ioutil.TempDir(os.TempDir(), "fuse-dest") - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["2.txt"] = fileInfo{0711, 333, 444, getRandomBtes(10)} - files["3.txt"] = fileInfo{0622, 333, 444, getRandomBtes(100)} - files["4.txt"] = fileInfo{0533, 333, 444, getRandomBtes(1024)} - files["5.txt"] = fileInfo{0544, 333, 444, getRandomBtes(10)} - files["6.txt"] = fileInfo{0555, 333, 444, getRandomBtes(10)} - files["7.txt"] = fileInfo{0666, 333, 444, getRandomBtes(10)} - files["8.txt"] = fileInfo{0777, 333, 333, getRandomBtes(10)} - files["11.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} - files["111.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} - files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} - files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)} - files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBtes(10)} - files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBtes(200)} - files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10240)} - files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["2.txt"] = fileInfo{0711, 333, 444, getRandomBytes(10)} + files["3.txt"] = fileInfo{0622, 333, 444, getRandomBytes(100)} + files["4.txt"] = fileInfo{0533, 333, 444, getRandomBytes(1024)} + files["5.txt"] = fileInfo{0544, 333, 444, getRandomBytes(10)} + files["6.txt"] = fileInfo{0555, 333, 444, getRandomBytes(10)} + files["7.txt"] = fileInfo{0666, 333, 444, getRandomBytes(10)} + files["8.txt"] = fileInfo{0777, 333, 333, getRandomBytes(10)} + files["11.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + files["111.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)} + files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBytes(10)} + files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBytes(200)} + files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10240)} + files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBytes(10)} - swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs.Stop() + for _, toEncrypt := range []bool{false, true} { + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - // Check unmount - _, err := swarmfs.Unmount(testMountDir) - if err != nil { - t.Fatalf("could not unmount %v", bzzHash) - } - if !isDirEmpty(testMountDir) { - t.Fatalf("unmount didnt work for %v", testMountDir) + swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs.Stop() + + // Check unmount + _, err := swarmfs.Unmount(testMountDir) + if err != nil { + t.Fatalf("could not unmount %v", bzzHash) + } + if !isDirEmpty(testMountDir) { + t.Fatalf("unmount didnt work for %v", testMountDir) + } } } func (ta *testAPI) maxMounts(t *testing.T) { + ta.runMaxMounts(false, t) + ta.runMaxMounts(true, t) +} + +func (ta *testAPI) runMaxMounts(toEncrypt bool, t *testing.T) { files := make(map[string]fileInfo) - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} uploadDir1, _ := ioutil.TempDir(os.TempDir(), "max-upload1") - bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1) + bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1, toEncrypt) mount1, _ := ioutil.TempDir(os.TempDir(), "max-mount1") swarmfs1 := mountDir(t, ta.api, files, bzzHash1, mount1) defer swarmfs1.Stop() - files["2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} uploadDir2, _ := ioutil.TempDir(os.TempDir(), "max-upload2") - bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2) + bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2, toEncrypt) mount2, _ := ioutil.TempDir(os.TempDir(), "max-mount2") swarmfs2 := mountDir(t, ta.api, files, bzzHash2, mount2) defer swarmfs2.Stop() - files["3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} uploadDir3, _ := ioutil.TempDir(os.TempDir(), "max-upload3") - bzzHash3 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir3) + bzzHash3 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir3, toEncrypt) mount3, _ := ioutil.TempDir(os.TempDir(), "max-mount3") swarmfs3 := mountDir(t, ta.api, files, bzzHash3, mount3) defer swarmfs3.Stop() - files["4.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["4.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} uploadDir4, _ := ioutil.TempDir(os.TempDir(), "max-upload4") - bzzHash4 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir4) + bzzHash4 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir4, toEncrypt) mount4, _ := ioutil.TempDir(os.TempDir(), "max-mount4") swarmfs4 := mountDir(t, ta.api, files, bzzHash4, mount4) defer swarmfs4.Stop() - files["5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} uploadDir5, _ := ioutil.TempDir(os.TempDir(), "max-upload5") - bzzHash5 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir5) + bzzHash5 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir5, toEncrypt) mount5, _ := ioutil.TempDir(os.TempDir(), "max-mount5") swarmfs5 := mountDir(t, ta.api, files, bzzHash5, mount5) defer swarmfs5.Stop() - files["6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} uploadDir6, _ := ioutil.TempDir(os.TempDir(), "max-upload6") - bzzHash6 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir6) + bzzHash6 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir6, toEncrypt) mount6, _ := ioutil.TempDir(os.TempDir(), "max-mount6") os.RemoveAll(mount6) @@ -278,527 +286,555 @@ func (ta *testAPI) maxMounts(t *testing.T) { if err == nil { t.Fatalf("Error: Going beyond max mounts %v", bzzHash6) } - } func (ta *testAPI) remount(t *testing.T) { - files := make(map[string]fileInfo) - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - uploadDir1, _ := ioutil.TempDir(os.TempDir(), "re-upload1") - bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1) - testMountDir1, _ := ioutil.TempDir(os.TempDir(), "re-mount1") - swarmfs := mountDir(t, ta.api, files, bzzHash1, testMountDir1) - defer swarmfs.Stop() + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + uploadDir1, _ := ioutil.TempDir(os.TempDir(), "re-upload1") + bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1, toEncrypt) + testMountDir1, _ := ioutil.TempDir(os.TempDir(), "re-mount1") + swarmfs := mountDir(t, ta.api, files, bzzHash1, testMountDir1) + defer swarmfs.Stop() - uploadDir2, _ := ioutil.TempDir(os.TempDir(), "re-upload2") - bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2) - testMountDir2, _ := ioutil.TempDir(os.TempDir(), "re-mount2") + uploadDir2, _ := ioutil.TempDir(os.TempDir(), "re-upload2") + bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2, toEncrypt) + testMountDir2, _ := ioutil.TempDir(os.TempDir(), "re-mount2") - // try mounting the same hash second time - os.RemoveAll(testMountDir2) - os.MkdirAll(testMountDir2, 0777) - _, err := swarmfs.Mount(bzzHash1, testMountDir2) - if err != nil { - t.Fatalf("Error mounting hash %v", bzzHash1) - } + // try mounting the same hash second time + os.RemoveAll(testMountDir2) + os.MkdirAll(testMountDir2, 0777) + _, err := swarmfs.Mount(bzzHash1, testMountDir2) + if err != nil { + t.Fatalf("Error mounting hash %v", bzzHash1) + } - // mount a different hash in already mounted point - _, err = swarmfs.Mount(bzzHash2, testMountDir1) - if err == nil { - t.Fatalf("Error mounting hash %v", bzzHash2) - } + // mount a different hash in already mounted point + _, err = swarmfs.Mount(bzzHash2, testMountDir1) + if err == nil { + t.Fatalf("Error mounting hash %v", bzzHash2) + } - // mount nonexistent hash - _, err = swarmfs.Mount("0xfea11223344", testMountDir1) - if err == nil { - t.Fatalf("Error mounting hash %v", bzzHash2) + // mount nonexistent hash + _, err = swarmfs.Mount("0xfea11223344", testMountDir1) + if err == nil { + t.Fatalf("Error mounting hash %v", bzzHash2) + } } } func (ta *testAPI) unmount(t *testing.T) { - files := make(map[string]fileInfo) - uploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + uploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount") - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir, toEncrypt) - swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs.Stop() + swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs.Stop() - swarmfs.Unmount(testMountDir) + swarmfs.Unmount(testMountDir) - mi := swarmfs.Listmounts() - for _, minfo := range mi { - if minfo.MountPoint == testMountDir { - t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir) + mi := swarmfs.Listmounts() + for _, minfo := range mi { + if minfo.MountPoint == testMountDir { + t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir) + } } } } func (ta *testAPI) unmountWhenResourceBusy(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount") - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs.Stop() + swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs.Stop() - actualPath := filepath.Join(testMountDir, "2.txt") - d, err := os.OpenFile(actualPath, os.O_RDWR, os.FileMode(0700)) - d.Write(getRandomBtes(10)) + actualPath := filepath.Join(testMountDir, "2.txt") + d, err := os.OpenFile(actualPath, os.O_RDWR, os.FileMode(0700)) + d.Write(getRandomBytes(10)) - _, err = swarmfs.Unmount(testMountDir) - if err != nil { - t.Fatalf("could not unmount %v", bzzHash) - } - d.Close() + _, err = swarmfs.Unmount(testMountDir) + if err != nil { + t.Fatalf("could not unmount %v", bzzHash) + } + d.Close() - mi := swarmfs.Listmounts() - for _, minfo := range mi { - if minfo.MountPoint == testMountDir { - t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir) + mi := swarmfs.Listmounts() + for _, minfo := range mi { + if minfo.MountPoint == testMountDir { + t.Fatalf("mount state not cleaned up in unmount case %v", testMountDir) + } } } } func (ta *testAPI) seekInMultiChunkFile(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "seek-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "seek-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "seek-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "seek-mount") - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10240)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10240)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs.Stop() + swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs.Stop() - // Create a new file seek the second chunk - actualPath := filepath.Join(testMountDir, "1.txt") - d, _ := os.OpenFile(actualPath, os.O_RDONLY, os.FileMode(0700)) + // Create a new file seek the second chunk + actualPath := filepath.Join(testMountDir, "1.txt") + d, _ := os.OpenFile(actualPath, os.O_RDONLY, os.FileMode(0700)) - d.Seek(5000, 0) + d.Seek(5000, 0) - contents := make([]byte, 1024) - d.Read(contents) - finfo := files["1.txt"] + contents := make([]byte, 1024) + d.Read(contents) + finfo := files["1.txt"] - if !bytes.Equal(finfo.contents[:6024][5000:], contents) { - t.Fatalf("File seek contents mismatch") + if !bytes.Equal(finfo.contents[:6024][5000:], contents) { + t.Fatalf("File seek contents mismatch") + } + d.Close() } - d.Close() } func (ta *testAPI) createNewFile(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "create-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "create-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "create-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "create-mount") - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - // Create a new file in the root dir and check - actualPath := filepath.Join(testMountDir, "2.txt") - d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) - if err1 != nil { - t.Fatalf("Could not create file %s : %v", actualPath, err1) + // Create a new file in the root dir and check + actualPath := filepath.Join(testMountDir, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["2.txt"] = fileInfo{0700, 333, 444, contents} + swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "2.txt", contents) } - contents := make([]byte, 11) - rand.Read(contents) - d.Write(contents) - d.Close() - - mi, err2 := swarmfs1.Unmount(testMountDir) - if err2 != nil { - t.Fatalf("Could not unmount %v", err2) - } - - // mount again and see if things are okay - files["2.txt"] = fileInfo{0700, 333, 444, contents} - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() - - checkFile(t, testMountDir, "2.txt", contents) } func (ta *testAPI) createNewFileInsideDirectory(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-mount") - files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - // Create a new file inside a existing dir and check - dirToCreate := filepath.Join(testMountDir, "one") - actualPath := filepath.Join(dirToCreate, "2.txt") - d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) - if err1 != nil { - t.Fatalf("Could not create file %s : %v", actualPath, err1) + // Create a new file inside a existing dir and check + dirToCreate := filepath.Join(testMountDir, "one") + actualPath := filepath.Join(dirToCreate, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["one/2.txt"] = fileInfo{0700, 333, 444, contents} + swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "one/2.txt", contents) } - contents := make([]byte, 11) - rand.Read(contents) - d.Write(contents) - d.Close() - - mi, err2 := swarmfs1.Unmount(testMountDir) - if err2 != nil { - t.Fatalf("Could not unmount %v", err2) - } - - // mount again and see if things are okay - files["one/2.txt"] = fileInfo{0700, 333, 444, contents} - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() - - checkFile(t, testMountDir, "one/2.txt", contents) } func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-mount") - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - // Create a new file inside a existing dir and check - dirToCreate := filepath.Join(testMountDir, "one") - os.MkdirAll(dirToCreate, 0777) - actualPath := filepath.Join(dirToCreate, "2.txt") - d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) - if err1 != nil { - t.Fatalf("Could not create file %s : %v", actualPath, err1) + // Create a new file inside a existing dir and check + dirToCreate := filepath.Join(testMountDir, "one") + os.MkdirAll(dirToCreate, 0777) + actualPath := filepath.Join(dirToCreate, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() + + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["one/2.txt"] = fileInfo{0700, 333, 444, contents} + swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "one/2.txt", contents) } - contents := make([]byte, 11) - rand.Read(contents) - d.Write(contents) - d.Close() - - mi, err2 := swarmfs1.Unmount(testMountDir) - if err2 != nil { - t.Fatalf("Could not unmount %v", err2) - } - - // mount again and see if things are okay - files["one/2.txt"] = fileInfo{0700, 333, 444, contents} - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() - - checkFile(t, testMountDir, "one/2.txt", contents) } func (ta *testAPI) removeExistingFile(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount") - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - // Remove a file in the root dir and check - actualPath := filepath.Join(testMountDir, "five.txt") - os.Remove(actualPath) + // Remove a file in the root dir and check + actualPath := filepath.Join(testMountDir, "five.txt") + os.Remove(actualPath) - mi, err2 := swarmfs1.Unmount(testMountDir) - if err2 != nil { - t.Fatalf("Could not unmount %v", err2) + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + delete(files, "five.txt") + swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() } - - // mount again and see if things are okay - delete(files, "five.txt") - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() } func (ta *testAPI) removeExistingFileInsideDir(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount") - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - // Remove a file in the root dir and check - actualPath := filepath.Join(testMountDir, "one/five.txt") - os.Remove(actualPath) + // Remove a file in the root dir and check + actualPath := filepath.Join(testMountDir, "one/five.txt") + os.Remove(actualPath) - mi, err2 := swarmfs1.Unmount(testMountDir) - if err2 != nil { - t.Fatalf("Could not unmount %v", err2) + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + delete(files, "one/five.txt") + swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() } - - // mount again and see if things are okay - delete(files, "one/five.txt") - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() } func (ta *testAPI) removeNewlyAddedFile(t *testing.T) { + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "removenew-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "removenew-mount") - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "removenew-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "removenew-mount") + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + // Adda a new file and remove it + dirToCreate := filepath.Join(testMountDir, "one") + os.MkdirAll(dirToCreate, os.FileMode(0665)) + actualPath := filepath.Join(dirToCreate, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + contents := make([]byte, 11) + rand.Read(contents) + d.Write(contents) + d.Close() - // Adda a new file and remove it - dirToCreate := filepath.Join(testMountDir, "one") - os.MkdirAll(dirToCreate, os.FileMode(0665)) - actualPath := filepath.Join(dirToCreate, "2.txt") - d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) - if err1 != nil { - t.Fatalf("Could not create file %s : %v", actualPath, err1) - } - contents := make([]byte, 11) - rand.Read(contents) - d.Write(contents) - d.Close() + checkFile(t, testMountDir, "one/2.txt", contents) - checkFile(t, testMountDir, "one/2.txt", contents) + os.Remove(actualPath) - os.Remove(actualPath) + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } - mi, err2 := swarmfs1.Unmount(testMountDir) - if err2 != nil { - t.Fatalf("Could not unmount %v", err2) - } + // mount again and see if things are okay + swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() - // mount again and see if things are okay - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() - - if bzzHash != mi.LatestManifest { - t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest) + if bzzHash != mi.LatestManifest { + t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest) + } } } func (ta *testAPI) addNewFileAndModifyContents(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-mount") - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - // Create a new file in the root dir and check - actualPath := filepath.Join(testMountDir, "2.txt") - d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) - if err1 != nil { - t.Fatalf("Could not create file %s : %v", actualPath, err1) + // Create a new file in the root dir and check + actualPath := filepath.Join(testMountDir, "2.txt") + d, err1 := os.OpenFile(actualPath, os.O_RDWR|os.O_CREATE, os.FileMode(0665)) + if err1 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err1) + } + line1 := []byte("Line 1") + rand.Read(line1) + d.Write(line1) + d.Close() + + mi1, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v", err2) + } + + // mount again and see if things are okay + files["2.txt"] = fileInfo{0700, 333, 444, line1} + swarmfs2 := mountDir(t, ta.api, files, mi1.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "2.txt", line1) + + mi2, err3 := swarmfs2.Unmount(testMountDir) + if err3 != nil { + t.Fatalf("Could not unmount %v", err3) + } + + // mount again and modify + swarmfs3 := mountDir(t, ta.api, files, mi2.LatestManifest, testMountDir) + defer swarmfs3.Stop() + + fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) + if err4 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err4) + } + line2 := []byte("Line 2") + rand.Read(line2) + fd.Seek(int64(len(line1)), 0) + fd.Write(line2) + fd.Close() + + mi3, err5 := swarmfs3.Unmount(testMountDir) + if err5 != nil { + t.Fatalf("Could not unmount %v", err5) + } + + // mount again and see if things are okay + b := [][]byte{line1, line2} + line1and2 := bytes.Join(b, []byte("")) + files["2.txt"] = fileInfo{0700, 333, 444, line1and2} + swarmfs4 := mountDir(t, ta.api, files, mi3.LatestManifest, testMountDir) + defer swarmfs4.Stop() + + checkFile(t, testMountDir, "2.txt", line1and2) } - line1 := []byte("Line 1") - rand.Read(line1) - d.Write(line1) - d.Close() - - mi1, err2 := swarmfs1.Unmount(testMountDir) - if err2 != nil { - t.Fatalf("Could not unmount %v", err2) - } - - // mount again and see if things are okay - files["2.txt"] = fileInfo{0700, 333, 444, line1} - swarmfs2 := mountDir(t, ta.api, files, mi1.LatestManifest, testMountDir) - defer swarmfs2.Stop() - - checkFile(t, testMountDir, "2.txt", line1) - - mi2, err3 := swarmfs2.Unmount(testMountDir) - if err3 != nil { - t.Fatalf("Could not unmount %v", err3) - } - - // mount again and modify - swarmfs3 := mountDir(t, ta.api, files, mi2.LatestManifest, testMountDir) - defer swarmfs3.Stop() - - fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) - if err4 != nil { - t.Fatalf("Could not create file %s : %v", actualPath, err4) - } - line2 := []byte("Line 2") - rand.Read(line2) - fd.Seek(int64(len(line1)), 0) - fd.Write(line2) - fd.Close() - - mi3, err5 := swarmfs3.Unmount(testMountDir) - if err5 != nil { - t.Fatalf("Could not unmount %v", err5) - } - - // mount again and see if things are okay - b := [][]byte{line1, line2} - line1and2 := bytes.Join(b, []byte("")) - files["2.txt"] = fileInfo{0700, 333, 444, line1and2} - swarmfs4 := mountDir(t, ta.api, files, mi3.LatestManifest, testMountDir) - defer swarmfs4.Stop() - - checkFile(t, testMountDir, "2.txt", line1and2) } func (ta *testAPI) removeEmptyDir(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount") - files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - os.MkdirAll(filepath.Join(testMountDir, "newdir"), 0777) + os.MkdirAll(filepath.Join(testMountDir, "newdir"), 0777) - mi, err3 := swarmfs1.Unmount(testMountDir) - if err3 != nil { - t.Fatalf("Could not unmount %v", err3) - } - if bzzHash != mi.LatestManifest { - t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest) + mi, err3 := swarmfs1.Unmount(testMountDir) + if err3 != nil { + t.Fatalf("Could not unmount %v", err3) + } + if bzzHash != mi.LatestManifest { + t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest) + } } } func (ta *testAPI) removeDirWhichHasFiles(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount") - files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - dirPath := filepath.Join(testMountDir, "two") - os.RemoveAll(dirPath) + dirPath := filepath.Join(testMountDir, "two") + os.RemoveAll(dirPath) - mi, err2 := swarmfs1.Unmount(testMountDir) - if err2 != nil { - t.Fatalf("Could not unmount %v ", err2) + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v ", err2) + } + + // mount again and see if things are okay + delete(files, "two/five.txt") + delete(files, "two/six.txt") + + swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() } - - // mount again and see if things are okay - delete(files, "two/five.txt") - delete(files, "two/six.txt") - - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() } func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-mount") - files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} - files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)} + files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} + files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - dirPath := filepath.Join(testMountDir, "two") - os.RemoveAll(dirPath) + dirPath := filepath.Join(testMountDir, "two") + os.RemoveAll(dirPath) - mi, err2 := swarmfs1.Unmount(testMountDir) - if err2 != nil { - t.Fatalf("Could not unmount %v ", err2) + mi, err2 := swarmfs1.Unmount(testMountDir) + if err2 != nil { + t.Fatalf("Could not unmount %v ", err2) + } + + // mount again and see if things are okay + delete(files, "two/three/2.txt") + delete(files, "two/three/3.txt") + delete(files, "two/four/5.txt") + delete(files, "two/four/6.txt") + delete(files, "two/four/six/7.txt") + + swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) + defer swarmfs2.Stop() } - - // mount again and see if things are okay - delete(files, "two/three/2.txt") - delete(files, "two/three/3.txt") - delete(files, "two/four/5.txt") - delete(files, "two/four/6.txt") - delete(files, "two/four/six/7.txt") - - swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir) - defer swarmfs2.Stop() } func (ta *testAPI) appendFileContentsToEnd(t *testing.T) { - files := make(map[string]fileInfo) - testUploadDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-upload") - testMountDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-mount") + for _, toEncrypt := range []bool{false, true} { + files := make(map[string]fileInfo) + testUploadDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-upload") + testMountDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-mount") - line1 := make([]byte, 10) - rand.Read(line1) - files["1.txt"] = fileInfo{0700, 333, 444, line1} - bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir) + line1 := make([]byte, 10) + rand.Read(line1) + files["1.txt"] = fileInfo{0700, 333, 444, line1} + bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt) - swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) - defer swarmfs1.Stop() + swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir) + defer swarmfs1.Stop() - actualPath := filepath.Join(testMountDir, "1.txt") - fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) - if err4 != nil { - t.Fatalf("Could not create file %s : %v", actualPath, err4) + actualPath := filepath.Join(testMountDir, "1.txt") + fd, err4 := os.OpenFile(actualPath, os.O_RDWR|os.O_APPEND, os.FileMode(0665)) + if err4 != nil { + t.Fatalf("Could not create file %s : %v", actualPath, err4) + } + line2 := make([]byte, 5) + rand.Read(line2) + fd.Seek(int64(len(line1)), 0) + fd.Write(line2) + fd.Close() + + mi1, err5 := swarmfs1.Unmount(testMountDir) + if err5 != nil { + t.Fatalf("Could not unmount %v ", err5) + } + + // mount again and see if things are okay + b := [][]byte{line1, line2} + line1and2 := bytes.Join(b, []byte("")) + files["1.txt"] = fileInfo{0700, 333, 444, line1and2} + swarmfs2 := mountDir(t, ta.api, files, mi1.LatestManifest, testMountDir) + defer swarmfs2.Stop() + + checkFile(t, testMountDir, "1.txt", line1and2) } - line2 := make([]byte, 5) - rand.Read(line2) - fd.Seek(int64(len(line1)), 0) - fd.Write(line2) - fd.Close() - - mi1, err5 := swarmfs1.Unmount(testMountDir) - if err5 != nil { - t.Fatalf("Could not unmount %v ", err5) - } - - // mount again and see if things are okay - b := [][]byte{line1, line2} - line1and2 := bytes.Join(b, []byte("")) - files["1.txt"] = fileInfo{0700, 333, 444, line1and2} - swarmfs2 := mountDir(t, ta.api, files, mi1.LatestManifest, testMountDir) - defer swarmfs2.Stop() - - checkFile(t, testMountDir, "1.txt", line1and2) } func TestFUSE(t *testing.T) { diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index bbdf710483..af6e2a06ac 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -100,3 +100,7 @@ func (self *DPA) Store(data io.Reader, size int64, toEncrypt bool) (key Key, wai putter := NewHasherStore(self.ChunkStore, self.hashFunc, toEncrypt) return PyramidSplit(data, putter, putter) } + +func (self *DPA) HashSize() int { + return self.hashFunc().Size() +} From 3e304ce36c03216dd4f09af06f31a9c5ea51ef69 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 5 Apr 2018 14:58:17 +0200 Subject: [PATCH 02/10] swarm/api: Implement encrypted manifest creation and update --- swarm/api/api.go | 2 +- swarm/api/filesystem.go | 2 +- swarm/api/http/server.go | 11 ++++++++--- swarm/api/http/server_test.go | 4 ++-- swarm/api/manifest.go | 37 ++++++++++++++++++----------------- swarm/api/manifest_test.go | 12 +++++++++--- 6 files changed, 40 insertions(+), 28 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 57b6300cf7..07c29a29d2 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -379,7 +379,7 @@ func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) apiModifyFail.Inc(1) return nil, err } - return trie.hash, nil + return trie.ref, nil } func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) { diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index e816a17c92..cd682e507d 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -164,7 +164,7 @@ func (self *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, err err2 := trie.recalcAndStore() var hs string if err2 == nil { - hs = trie.hash.Hex() + hs = trie.ref.Hex() } awg.Wait() return hs, err2 diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 6b7e44437e..73dd0ca13f 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -177,8 +177,13 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { return } + toEncrypt := false + if r.uri.Addr == "encrypt" { + toEncrypt = true + } + var key storage.Key - if r.uri.Addr != "" { + if r.uri.Addr != "" && r.uri.Addr != "encrypt" { key, err = s.api.Resolve(r.uri) if err != nil { postFilesFail.Inc(1) @@ -187,7 +192,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { } log.Debug("resolved key", "ruid", r.ruid, "key", key) } else { - key, err = s.api.NewManifest(false) + key, err = s.api.NewManifest(toEncrypt) if err != nil { postFilesFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) @@ -376,7 +381,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { Respond(w, r, err2.Error(), code) return } - m, err := s.api.NewResourceManifest(r.uri.Addr, false) + m, err := s.api.NewResourceManifest(r.uri.Addr) if err != nil { Respond(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError) return diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index b94847f82d..5f87ab6537 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -218,8 +218,8 @@ func TestBzzResource(t *testing.T) { } func TestBzzGetPath(t *testing.T) { - // testBzzGetPath(false, t) - testBzzGetPath(true, t) + testBzzGetPath(false, t) + // testBzzGetPath(true, t) } func testBzzGetPath(encrypted bool, t *testing.T) { diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 6047066a64..4b3d72909c 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -72,7 +72,7 @@ func (a *Api) NewManifest(toEncrypt bool) (storage.Key, error) { // Manifest hack for supporting Mutable Resource Updates from the bzz: scheme // see swarm/api/api.go:Api.Get() for more information -func (a *Api) NewResourceManifest(resourceKey string, toEncrypt bool) (storage.Key, error) { +func (a *Api) NewResourceManifest(resourceKey string) (storage.Key, error) { var manifest Manifest entry := ManifestEntry{ Hash: resourceKey, @@ -83,7 +83,7 @@ func (a *Api) NewResourceManifest(resourceKey string, toEncrypt bool) (storage.K if err != nil { return nil, err } - key, _, err := a.Store(bytes.NewReader(data), int64(len(data)), toEncrypt) + key, _, err := a.Store(bytes.NewReader(data), int64(len(data)), false) return key, err } @@ -105,9 +105,7 @@ func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWrit // AddEntry stores the given data and adds the resulting key to the manifest func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) { - toEncrypt := (len(m.trie.hash) > m.trie.dpa.HashSize()) - - key, _, err := m.api.Store(data, e.Size, toEncrypt) + key, _, err := m.api.Store(data, e.Size, m.trie.encrypted) if err != nil { return nil, err } @@ -125,7 +123,7 @@ func (m *ManifestWriter) RemoveEntry(path string) error { // Store stores the manifest, returning the resulting storage key func (m *ManifestWriter) Store() (storage.Key, error) { - return m.trie.hash, m.trie.recalcAndStore() + return m.trie.ref, m.trie.recalcAndStore() } // ManifestWalker is used to recursively walk the entries in the manifest and @@ -185,9 +183,10 @@ func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) } type manifestTrie struct { - dpa *storage.DPA - entries [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry - hash storage.Key // if hash != nil, it is stored + dpa *storage.DPA + entries [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry + ref storage.Key // if ref != nil, it is stored + encrypted bool } func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry { @@ -231,7 +230,7 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp return } - log.Trace("manifest retrieved", "key", hash) + log.Debug("manifest retrieved", "key", hash) var man struct { Entries []*manifestTrieEntry `json:"entries"` } @@ -245,7 +244,8 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp log.Trace("manifest entries", "key", hash, "len", len(man.Entries)) trie = &manifestTrie{ - dpa: dpa, + dpa: dpa, + encrypted: (len(hash) > dpa.HashSize()), } for _, entry := range man.Entries { trie.addEntry(entry, quitC) @@ -254,7 +254,7 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp } func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { - self.hash = nil // trie modified, hash needs to be re-calculated on demand + self.ref = nil // trie modified, hash needs to be re-calculated on demand if len(entry.Path) == 0 { self.entries[256] = entry @@ -286,7 +286,8 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { commonPrefix := entry.Path[:cpl] subtrie := &manifestTrie{ - dpa: self.dpa, + dpa: self.dpa, + encrypted: self.encrypted, } entry.Path = entry.Path[cpl:] oldentry.Path = oldentry.Path[cpl:] @@ -310,7 +311,7 @@ func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { } func (self *manifestTrie) deleteEntry(path string, quitC chan bool) { - self.hash = nil // trie modified, hash needs to be re-calculated on demand + self.ref = nil // trie modified, hash needs to be re-calculated on demand if len(path) == 0 { self.entries[256] = nil @@ -346,7 +347,7 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) { } func (self *manifestTrie) recalcAndStore() error { - if self.hash != nil { + if self.ref != nil { return nil } @@ -361,7 +362,7 @@ func (self *manifestTrie) recalcAndStore() error { if err != nil { return err } - entry.Hash = entry.subtrie.hash.Hex() + entry.Hash = entry.subtrie.ref.Hex() } list.Entries = append(list.Entries, entry.ManifestEntry) } @@ -374,9 +375,9 @@ func (self *manifestTrie) recalcAndStore() error { } sr := bytes.NewReader(manifest) - key, wait, err2 := self.dpa.Store(sr, int64(len(manifest)), false) + key, wait, err2 := self.dpa.Store(sr, int64(len(manifest)), self.encrypted) wait() - self.hash = key + self.ref = key return err2 } diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index 7098ca16fd..27bf32007a 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -42,7 +42,9 @@ func manifest(paths ...string) (manifestReader storage.LazySectionReader) { func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...string) *manifestTrie { quitC := make(chan bool) - trie, err := readManifest(manifest(paths...), nil, nil, quitC) + dpa := storage.NewDPA(nil, storage.NewDPAParams()) + ref := make([]byte, dpa.HashSize()) + trie, err := readManifest(manifest(paths...), ref, dpa, quitC) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } @@ -97,7 +99,9 @@ func TestGetEntry(t *testing.T) { func TestExactMatch(t *testing.T) { quitC := make(chan bool) mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map") - trie, err := readManifest(mf, nil, nil, quitC) + dpa := storage.NewDPA(nil, storage.NewDPAParams()) + ref := make([]byte, dpa.HashSize()) + trie, err := readManifest(mf, ref, dpa, quitC) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } @@ -128,7 +132,9 @@ func TestAddFileWithManifestPath(t *testing.T) { reader := &storage.LazyTestSectionReader{ SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))), } - trie, err := readManifest(reader, nil, nil, nil) + dpa := storage.NewDPA(nil, storage.NewDPAParams()) + ref := make([]byte, dpa.HashSize()) + trie, err := readManifest(reader, ref, dpa, nil) if err != nil { t.Fatal(err) } From 221188af08a52eceb1d1eddc7c239bc867ce0fa7 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 6 Apr 2018 16:21:54 +0200 Subject: [PATCH 03/10] swarm/api: Fix linter --- swarm/api/api.go | 1 - 1 file changed, 1 deletion(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 07c29a29d2..c7cb9c0662 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -39,7 +39,6 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) - // TODO: this is bad, it should not be hardcoded how long is a hash var hashMatcher = regexp.MustCompile("^([0-9A-Fa-f]{64})([0-9A-Fa-f]{64})?") From 661591e0a6cabab0d9d894af687a5d83c69741ca Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Fri, 6 Apr 2018 16:22:48 +0200 Subject: [PATCH 04/10] swarm/api/http: Add comment about missing test --- swarm/api/http/server_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 5f87ab6537..6ec72330cb 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -218,8 +218,8 @@ func TestBzzResource(t *testing.T) { } func TestBzzGetPath(t *testing.T) { + // TODO: add encrypted test case testBzzGetPath(false, t) - // testBzzGetPath(true, t) } func testBzzGetPath(encrypted bool, t *testing.T) { From c63b03681233714316c3b7c508d5c321d119287f Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Wed, 4 Apr 2018 02:47:22 +0000 Subject: [PATCH 05/10] pss: remove expired entries from forward cache --- swarm/pss/pss.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 1b88c95db2..fed71a389b 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -756,6 +756,17 @@ func (self *Pss) forward(msg *PssMsg) { // SECTION: Caching ///////////////////////////////////////////////////////////////////// +// remove expired entries from forward cache +func (self *Pss) cleanFwdCache() { + self.fwdCacheMu.Lock() + defer self.fwdCacheMu.Unlock() + for k,v := range self.fwdCache { + if v.expiresAt.Before(time.Now()) { + delete(self.fwdCache[k]) + } + } +} + // add a message to the cache func (self *Pss) addFwdCache(msg *PssMsg) error { var entry pssCacheEntry From 8054d2f3a603a3426f00649320cb98b5694050c1 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Wed, 4 Apr 2018 21:26:14 +0000 Subject: [PATCH 06/10] format. Add separate tick for cache cleaning --- swarm/pss/pss.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index fed71a389b..267620ce00 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -177,7 +177,10 @@ func (self *Pss) Start(srv *p2p.Server) error { go func() { for { tickC := time.Tick(defaultCleanInterval) + cacheTickC := time.Tick(cacheTTL) select { + case <-cacheTickC: + self.cleanFwdCache() case <-tickC: self.cleanKeys() case <-self.quitC: @@ -758,13 +761,13 @@ func (self *Pss) forward(msg *PssMsg) { // remove expired entries from forward cache func (self *Pss) cleanFwdCache() { - self.fwdCacheMu.Lock() - defer self.fwdCacheMu.Unlock() - for k,v := range self.fwdCache { - if v.expiresAt.Before(time.Now()) { - delete(self.fwdCache[k]) - } - } + self.fwdCacheMu.Lock() + defer self.fwdCacheMu.Unlock() + for k, v := range self.fwdCache { + if v.expiresAt.Before(time.Now()) { + delete(self.fwdCache[k]) + } + } } // add a message to the cache From 83355516c302b54c371eb41d296117846f316103 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Thu, 5 Apr 2018 00:15:41 +0000 Subject: [PATCH 07/10] fix errors --- swarm/pss/pss.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 267620ce00..327f2bd46d 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -177,7 +177,7 @@ func (self *Pss) Start(srv *p2p.Server) error { go func() { for { tickC := time.Tick(defaultCleanInterval) - cacheTickC := time.Tick(cacheTTL) + cacheTickC := time.Tick(self.cacheTTL) select { case <-cacheTickC: self.cleanFwdCache() @@ -765,7 +765,7 @@ func (self *Pss) cleanFwdCache() { defer self.fwdCacheMu.Unlock() for k, v := range self.fwdCache { if v.expiresAt.Before(time.Now()) { - delete(self.fwdCache[k]) + delete(self.fwdCache, k) } } } From 04367fc5ea2434f721234744716bf77fc767a52f Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Fri, 6 Apr 2018 07:11:21 +0000 Subject: [PATCH 08/10] pss: add testing for forwarding cache clearing. --- swarm/pss/pss_test.go | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 254fa5c09e..5a6648982d 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -144,9 +144,11 @@ func TestCache(t *testing.T) { t.Fatal(err) } ps := newTestPss(privkey, nil, nil) + pp := NewPssParams(privkey) data := []byte("foo") datatwo := []byte("bar") + datathree := []byte("baz") wparams := &whisper.MessageParams{ TTL: defaultWhisperTTL, Src: privkey, @@ -169,6 +171,13 @@ func TestCache(t *testing.T) { Payload: envtwo, To: to, } + wparams.Payload = datathree + woutmsg, err = whisper.NewSentMessage(wparams) + envthree, err := woutmsg.Wrap(wparams) + msgthree := &PssMsg{ + Payload: envthree, + To: to, + } digest := ps.digest(msg) if err != nil { @@ -178,6 +187,11 @@ func TestCache(t *testing.T) { if err != nil { t.Fatalf("could not store cache msgtwo: %v", err) } + digestthree := ps.digest(msgthree) + if err != nil { + t.Fatalf("could not store cache msgthree: %v", err) + } + if digest == digesttwo { t.Fatalf("different msgs return same hash: %d", digesttwo) @@ -197,10 +211,23 @@ func TestCache(t *testing.T) { t.Fatalf("message %v should NOT have EXPIRE record in cache but checkCache returned true", msgtwo) } - time.Sleep(pp.CacheTTL) + time.Sleep(pp.CacheTTL + 1*time.Second) + err = ps.addFwdCache(msgthree) + if err != nil { + t.Fatalf("write to pss expire cache failed: %v", err) + } + if ps.checkFwdCache(msg) { t.Fatalf("message %v should have expired from cache but checkCache returned true", msg) } + + if _, ok := ps.fwdCache[digestthree]; !ok { + t.Fatalf("unexpired message should be in the cache: %v", digestthree) + } + + if _, ok := ps.fwdCache[digesttwo]; ok { + t.Fatalf("expired message should have been cleared from the cache: %v", digesttwo) + } } // matching of address hints; whether a message could be or is for the node @@ -1309,6 +1336,7 @@ func newTestPss(privkey *ecdsa.PrivateKey, overlay network.Overlay, ppextra *Pss pp.SymKeyCacheCapacity = ppextra.SymKeyCacheCapacity } ps := NewPss(overlay, pp) + ps.Start(nil) return ps } From ac39a0ccb19337dc2e36669435af8fbb2f2640f5 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Mon, 9 Apr 2018 06:57:37 +0000 Subject: [PATCH 09/10] add documentation. fix formatting --- swarm/pss/pss.go | 2 +- swarm/pss/pss_test.go | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 327f2bd46d..3a91b546d5 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -759,7 +759,7 @@ func (self *Pss) forward(msg *PssMsg) { // SECTION: Caching ///////////////////////////////////////////////////////////////////// -// remove expired entries from forward cache +// cleanFwdCache is used to periodically remove expired entries from the forward cache func (self *Pss) cleanFwdCache() { self.fwdCacheMu.Lock() defer self.fwdCacheMu.Unlock() diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 5a6648982d..9eaba00ccc 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -144,11 +144,10 @@ func TestCache(t *testing.T) { t.Fatal(err) } ps := newTestPss(privkey, nil, nil) - pp := NewPssParams(privkey) data := []byte("foo") datatwo := []byte("bar") - datathree := []byte("baz") + datathree := []byte("baz") wparams := &whisper.MessageParams{ TTL: defaultWhisperTTL, Src: privkey, @@ -192,7 +191,6 @@ func TestCache(t *testing.T) { t.Fatalf("could not store cache msgthree: %v", err) } - if digest == digesttwo { t.Fatalf("different msgs return same hash: %d", digesttwo) } @@ -212,7 +210,7 @@ func TestCache(t *testing.T) { } time.Sleep(pp.CacheTTL + 1*time.Second) - err = ps.addFwdCache(msgthree) + err = ps.addFwdCache(msgthree) if err != nil { t.Fatalf("write to pss expire cache failed: %v", err) } @@ -221,9 +219,9 @@ func TestCache(t *testing.T) { t.Fatalf("message %v should have expired from cache but checkCache returned true", msg) } - if _, ok := ps.fwdCache[digestthree]; !ok { + if _, ok := ps.fwdCache[digestthree]; !ok { t.Fatalf("unexpired message should be in the cache: %v", digestthree) - } + } if _, ok := ps.fwdCache[digesttwo]; ok { t.Fatalf("expired message should have been cleared from the cache: %v", digesttwo) From 68b9ff136a84acfac7d94e782ce2ece4a76ce953 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Mon, 9 Apr 2018 12:23:35 +0200 Subject: [PATCH 10/10] swarm/api/http: Make server_test work with encryption Replace hardcoded hashes with dynamic references, because encrypted references are not deterministic --- swarm/api/http/server_test.go | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 6ec72330cb..901db68bc4 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -29,7 +29,6 @@ import ( "strings" "testing" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" @@ -218,23 +217,23 @@ func TestBzzResource(t *testing.T) { } func TestBzzGetPath(t *testing.T) { - // TODO: add encrypted test case testBzzGetPath(false, t) + testBzzGetPath(true, t) } func testBzzGetPath(encrypted bool, t *testing.T) { var err error testmanifest := []string{ - `{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`, - `{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`, `{"entries":[{"path":"b","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"c","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0}]}`, + `{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"","contentType":"application/bzz-manifest+json","status":0}]}`, + `{"entries":[{"path":"a/","hash":"","contentType":"application/bzz-manifest+json","status":0}]}`, } testrequests := make(map[string]int) - testrequests["/"] = 0 + testrequests["/"] = 2 testrequests["/a/"] = 1 - testrequests["/a/b/"] = 2 + testrequests["/a/b/"] = 0 testrequests["/x"] = 0 testrequests[""] = 0 @@ -251,13 +250,19 @@ func testBzzGetPath(encrypted bool, t *testing.T) { reader[i] = bytes.NewReader([]byte(mf)) var wait func() key[i], wait, err = srv.Dpa.Store(reader[i], int64(len(mf)), encrypted) + for j := i + 1; j < len(testmanifest); j++ { + testmanifest[j] = strings.Replace(testmanifest[j], fmt.Sprintf("", i), key[i].Hex(), -1) + } if err != nil { t.Fatal(err) } wait() + fmt.Println("!!!!!!!!!!", i, key[i]) } - _, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a") + rootRef := key[2].Hex() + + _, err = http.Get(srv.URL + "/bzz-raw:/" + rootRef + "/a") if err != nil { t.Fatalf("Failed to connect to proxy: %v", err) } @@ -268,7 +273,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { url := srv.URL + "/bzz-raw:/" if k[:] != "" { - url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain" + url += rootRef + "/" + k[1:] + "?content_type=text/plain" } resp, err = http.Get(url) if err != nil { @@ -297,7 +302,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { url := srv.URL + "/bzz-hash:/" if k[:] != "" { - url += common.ToHex(key[0])[2:] + "/" + k[1:] + url += rootRef + "/" + k[1:] } resp, err = http.Get(url) if err != nil { @@ -323,6 +328,8 @@ func testBzzGetPath(encrypted bool, t *testing.T) { } } + ref := key[2].Hex() + for _, c := range []struct { path string json string @@ -331,17 +338,17 @@ func testBzzGetPath(encrypted bool, t *testing.T) { { path: "/", json: `{"common_prefixes":["a/"]}`, - html: "\n\n\n \n \n\t\t\n\tSwarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/\n\n\n\n

Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\t\n\t \n\t \n\t \n\t\n \n\n \n
PathTypeSize
a/DIR-
\n
\n\n", + html: fmt.Sprintf("\n\n\n \n \n\t\t\n\tSwarm index of bzz:/%s/\n\n\n\n

Swarm index of bzz:/%s/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\t\n\t \n\t \n\t \n\t\n \n\n \n
PathTypeSize
a/DIR-
\n
\n\n", ref, ref), }, { path: "/a/", json: `{"common_prefixes":["a/b/"],"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/a","mod_time":"0001-01-01T00:00:00Z"}]}`, - html: "\n\n\n \n \n\t\t\n\tSwarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/\n\n\n\n

Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\t\n\t \n\t \n\t \n\t\n \n\n \n\t\n\t \n\t \n\t \n\t\n \n
PathTypeSize
b/DIR-
a0
\n
\n\n", + html: fmt.Sprintf("\n\n\n \n \n\t\t\n\tSwarm index of bzz:/%s/a/\n\n\n\n

Swarm index of bzz:/%s/a/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\t\n\t \n\t \n\t \n\t\n \n\n \n\t\n\t \n\t \n\t \n\t\n \n
PathTypeSize
b/DIR-
a0
\n
\n\n", ref, ref), }, { path: "/a/b/", json: `{"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/b","mod_time":"0001-01-01T00:00:00Z"},{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/c","mod_time":"0001-01-01T00:00:00Z"}]}`, - html: "\n\n\n \n \n\t\t\n\tSwarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/\n\n\n\n

Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\n \n\t\n\t \n\t \n\t \n\t\n \n\t\n\t \n\t \n\t \n\t\n \n
PathTypeSize
b0
c0
\n
\n\n", + html: fmt.Sprintf("\n\n\n \n \n\t\t\n\tSwarm index of bzz:/%s/a/b/\n\n\n\n

Swarm index of bzz:/%s/a/b/

\n
\n \n \n \n\t\n\t\n\t\n \n \n\n \n \n\n \n\t\n\t \n\t \n\t \n\t\n \n\t\n\t \n\t \n\t \n\t\n \n
PathTypeSize
b0
c0
\n
\n\n", ref, ref), }, { path: "/x", @@ -353,7 +360,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { k := c.path url := srv.URL + "/bzz-list:/" if k[:] != "" { - url += common.ToHex(key[0])[2:] + "/" + k[1:] + url += rootRef + "/" + k[1:] } t.Run("json list "+c.path, func(t *testing.T) { resp, err := http.Get(url)