From c342a8c376513c2962999a86580ff83cf0c09133 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Tue, 10 Apr 2018 13:54:58 +0200 Subject: [PATCH] swarm/...: There is a response header to show if content was encrypted DPA.Retrieve() also returns this information --- swarm/api/api.go | 6 +++--- swarm/api/client/client_test.go | 6 +++--- swarm/api/filesystem.go | 2 +- swarm/api/http/server.go | 7 +++++-- swarm/api/http/server_test.go | 2 +- swarm/api/manifest.go | 8 ++++---- swarm/api/manifest_test.go | 6 +++--- swarm/fuse/fuse_file.go | 4 ++-- swarm/network/stream/common_test.go | 2 +- swarm/storage/dpa.go | 9 ++++++--- swarm/storage/dpa_test.go | 25 ++++++++++++++++++++----- 11 files changed, 49 insertions(+), 28 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index c7cb9c0662..8b48419e44 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -238,7 +238,7 @@ func (self *Api) Upload(uploadDir, index string, toEncrypt bool) (hash string, e } // DPA reader API -func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader { +func (self *Api) Retrieve(key storage.Key) (reader storage.LazySectionReader, isEncrypted bool) { return self.dpa.Retrieve(key) } @@ -344,7 +344,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe } else { mimeType = entry.ContentType log.Trace("content lookup key", "key", key, "mimetype", mimeType) - reader = self.dpa.Retrieve(key) + reader, _ = self.dpa.Retrieve(key) } } else { status = http.StatusNotFound @@ -482,7 +482,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte buf := make([]byte, buffSize) - oldReader := self.Retrieve(oldKey) + oldReader, _ := self.Retrieve(oldKey) io.ReadAtLeast(oldReader, buf, int(offset)) newReader := bytes.NewReader(content) diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index c1d144e370..fb053ad662 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -74,7 +74,7 @@ func TestClientUploadDownloadFiles(t *testing.T) { Size: int64(len(data)), }, } - hash, err := client.Upload(file, manifest) + hash, err := client.Upload(file, manifest, false) if err != nil { t.Fatal(err) } @@ -168,7 +168,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) { // upload the directory client := NewClient(srv.URL) defaultPath := filepath.Join(dir, testDirFiles[0]) - hash, err := client.UploadDirectory(dir, defaultPath, "") + hash, err := client.UploadDirectory(dir, defaultPath, "", false) if err != nil { t.Fatalf("error uploading directory: %s", err) } @@ -224,7 +224,7 @@ func TestClientFileList(t *testing.T) { defer os.RemoveAll(dir) client := NewClient(srv.URL) - hash, err := client.UploadDirectory(dir, "", "") + hash, err := client.UploadDirectory(dir, "", "", false) if err != nil { t.Fatalf("error uploading directory: %s", err) } diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index cd682e507d..55497d9498 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -273,7 +273,7 @@ func retrieveToFile(quitC chan bool, dpa *storage.DPA, key storage.Key, path str if err != nil { return err } - reader := dpa.Retrieve(key) + reader, _ := dpa.Retrieve(key) writer := bufio.NewWriter(f) size, err := reader.Size(quitC) if err != nil { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 73dd0ca13f..644b859484 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -556,13 +556,15 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { } // check the root chunk exists by retrieving the file's size - reader := s.api.Retrieve(key) + reader, isEncrypted := s.api.Retrieve(key) if _, err := reader.Size(nil); err != nil { getFail.Inc(1) Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", key, err), http.StatusNotFound) return } + w.Header().Set("X-Encrypted", fmt.Sprintf("%v", isEncrypted)) + switch { case r.uri.Raw() || r.uri.DeprecatedRaw(): // allow the request to overwrite the content type using a query @@ -619,11 +621,12 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { } // retrieve the entry's key and size - reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash))) + reader, isEncrypted := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash))) size, err := reader.Size(nil) if err != nil { return err } + w.Header().Set("X-Encrypted", fmt.Sprintf("%v", isEncrypted)) // write a tar header for the entry hdr := &tar.Header{ diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 901db68bc4..78a2c77c68 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -472,7 +472,7 @@ func TestBzzRootRedirect(t *testing.T) { Size: int64(len(data)), }, } - hash, err := client.Upload(file, "") + hash, err := client.Upload(file, "", false) if err != nil { t.Fatal(err) } diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 4b3d72909c..239ec8a776 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -205,12 +205,12 @@ type manifestTrieEntry struct { func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand log.Trace("manifest lookup", "key", hash) // retrieve manifest via DPA - manifestReader := dpa.Retrieve(hash) + manifestReader, isEncrypted := dpa.Retrieve(hash) log.Trace("reader retrieved", "key", hash) - return readManifest(manifestReader, hash, dpa, quitC) + return readManifest(manifestReader, hash, dpa, isEncrypted, quitC) } -func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, isEncrypted bool, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand // TODO check size for oversized manifests size, err := manifestReader.Size(quitC) @@ -245,7 +245,7 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp trie = &manifestTrie{ dpa: dpa, - encrypted: (len(hash) > dpa.HashSize()), + encrypted: isEncrypted, } for _, entry := range man.Entries { trie.addEntry(entry, quitC) diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index 27bf32007a..fb8f943924 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -44,7 +44,7 @@ func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...stri quitC := make(chan bool) dpa := storage.NewDPA(nil, storage.NewDPAParams()) ref := make([]byte, dpa.HashSize()) - trie, err := readManifest(manifest(paths...), ref, dpa, quitC) + trie, err := readManifest(manifest(paths...), ref, dpa, false, quitC) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } @@ -101,7 +101,7 @@ func TestExactMatch(t *testing.T) { mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map") dpa := storage.NewDPA(nil, storage.NewDPAParams()) ref := make([]byte, dpa.HashSize()) - trie, err := readManifest(mf, ref, dpa, quitC) + trie, err := readManifest(mf, ref, dpa, false, quitC) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } @@ -134,7 +134,7 @@ func TestAddFileWithManifestPath(t *testing.T) { } dpa := storage.NewDPA(nil, storage.NewDPAParams()) ref := make([]byte, dpa.HashSize()) - trie, err := readManifest(reader, ref, dpa, nil) + trie, err := readManifest(reader, ref, dpa, false, nil) if err != nil { t.Fatal(err) } diff --git a/swarm/fuse/fuse_file.go b/swarm/fuse/fuse_file.go index c94a0773f5..41a401eed5 100644 --- a/swarm/fuse/fuse_file.go +++ b/swarm/fuse/fuse_file.go @@ -82,7 +82,7 @@ func (file *SwarmFile) Attr(ctx context.Context, a *fuse.Attr) error { a.Gid = uint32(os.Getegid()) if file.fileSize == -1 { - reader := file.mountInfo.swarmApi.Retrieve(file.key) + reader, _ := file.mountInfo.swarmApi.Retrieve(file.key) quitC := make(chan bool) size, err := reader.Size(quitC) if err != nil { @@ -99,7 +99,7 @@ func (sf *SwarmFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse sf.lock.RLock() defer sf.lock.RUnlock() if sf.reader == nil { - sf.reader = sf.mountInfo.swarmApi.Retrieve(sf.key) + sf.reader, _ = sf.mountInfo.swarmApi.Retrieve(sf.key) } buf := make([]byte, req.Size) n, err := sf.reader.ReadAt(buf, req.Offset) diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 241bd7fc3b..8428fd3fb4 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -216,7 +216,7 @@ func (r *TestRegistry) APIs() []rpc.API { } func readAll(dpa *storage.DPA, hash []byte) (int64, error) { - r := dpa.Retrieve(hash) + r, _ := dpa.Retrieve(hash) buf := make([]byte, 1024) var n int var total int64 diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index af6e2a06ac..8070db8a8b 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -89,9 +89,12 @@ func NewDPA(store ChunkStore, params *DPAParams) *DPA { // FS-aware API and httpaccess // Chunk retrieval blocks on netStore requests with a timeout so reader will // report error if retrieval of chunks within requested range time out. -func (self *DPA) Retrieve(key Key) LazySectionReader { - getter := NewHasherStore(self.ChunkStore, self.hashFunc, len(key) > self.hashFunc().Size()) - return TreeJoin(key, getter, 0) +// It returns a reader with the chunk data and whether the content was encrypted +func (self *DPA) Retrieve(key Key) (reader LazySectionReader, isEncrypted bool) { + isEncrypted = len(key) > self.hashFunc().Size() + getter := NewHasherStore(self.ChunkStore, self.hashFunc, isEncrypted) + reader = TreeJoin(key, getter, 0) + return } // Public API. Main entry point for document storage directly. Used by the diff --git a/swarm/storage/dpa_test.go b/swarm/storage/dpa_test.go index 1126f05a52..a134347638 100644 --- a/swarm/storage/dpa_test.go +++ b/swarm/storage/dpa_test.go @@ -54,7 +54,10 @@ func testDpaRandom(toEncrypt bool, t *testing.T) { t.Errorf("Store error: %v", err) } wait() - resultReader := dpa.Retrieve(key) + resultReader, isEncrypted := dpa.Retrieve(key) + if isEncrypted != toEncrypt { + t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) + } resultSlice := make([]byte, len(slice)) n, err := resultReader.ReadAt(resultSlice, 0) if err != io.EOF { @@ -69,7 +72,10 @@ func testDpaRandom(toEncrypt bool, t *testing.T) { ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666) ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666) localStore.memStore = NewMemStore(db, defaultCacheCapacity) - resultReader = dpa.Retrieve(key) + resultReader, isEncrypted = dpa.Retrieve(key) + if isEncrypted != toEncrypt { + t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) + } for i := range resultSlice { resultSlice[i] = 0 } @@ -109,7 +115,10 @@ func testDPA_capacity(toEncrypt bool, t *testing.T) { t.Errorf("Store error: %v", err) } wait() - resultReader := dpa.Retrieve(key) + resultReader, isEncrypted := dpa.Retrieve(key) + if isEncrypted != toEncrypt { + t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) + } resultSlice := make([]byte, len(slice)) n, err := resultReader.ReadAt(resultSlice, 0) if err != io.EOF { @@ -125,14 +134,20 @@ func testDPA_capacity(toEncrypt bool, t *testing.T) { memStore.setCapacity(0) // check whether it is, indeed, empty dpa.ChunkStore = memStore - resultReader = dpa.Retrieve(key) + resultReader, isEncrypted = dpa.Retrieve(key) + if isEncrypted != toEncrypt { + t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) + } if _, err = resultReader.ReadAt(resultSlice, 0); err == nil { t.Errorf("Was able to read %d bytes from an empty memStore.", len(slice)) } // check how it works with localStore dpa.ChunkStore = localStore // localStore.dbStore.setCapacity(0) - resultReader = dpa.Retrieve(key) + resultReader, isEncrypted = dpa.Retrieve(key) + if isEncrypted != toEncrypt { + t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) + } for i := range resultSlice { resultSlice[i] = 0 }