From 3d1a1aa459ed4ca003d966fe18f6e049b99b68c6 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Mon, 9 Apr 2018 23:52:13 +0200 Subject: [PATCH 1/7] cmd/swarm, swarm/api: Swarm command line api to upload encrypted files --- cmd/swarm/main.go | 11 ++++++++++- cmd/swarm/upload.go | 14 +++++++++++--- cmd/swarm/upload_test.go | 25 +++++++++++++++++++++---- swarm/api/client/client.go | 21 +++++++++++++++------ 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 057b032ce0..f4ff0bb731 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -218,12 +218,21 @@ The output of this command is supposed to be machine-readable. `, }, { - Action: upload, + Action: nonEncryptedUpload, Name: "up", Usage: "upload a file or directory to swarm using the HTTP API", ArgsUsage: " ", Description: ` "upload a file or directory to swarm using the HTTP API and prints the root hash", +`, + }, + { + Action: encryptedUpload, + Name: "encrypted-up", + Usage: "Upload a file or directory with encryption to swarm using the HTTP API. NOTE: Currently the reference for the uploaded content is non-deterministic, so you will receive different references if you upload it twice.", + ArgsUsage: " ", + Description: ` +"upload a file or directory to swarm using the HTTP API and prints the root hash", `, }, { diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 9f4c525bb9..058f93956d 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -35,7 +35,15 @@ import ( "gopkg.in/urfave/cli.v1" ) -func upload(ctx *cli.Context) { +func encryptedUpload(ctx *cli.Context) { + upload(ctx, true) +} + +func nonEncryptedUpload(ctx *cli.Context) { + upload(ctx, false) +} + +func upload(ctx *cli.Context, toEncrypt bool) { args := ctx.Args() var ( @@ -97,7 +105,7 @@ func upload(ctx *cli.Context) { if !recursive { return "", errors.New("Argument is a directory and recursive upload is disabled") } - return client.UploadDirectory(file, defaultPath, "") + return client.UploadDirectory(file, defaultPath, "", toEncrypt) } } else { doUpload = func() (string, error) { @@ -110,7 +118,7 @@ func upload(ctx *cli.Context) { mimeType = detectMimeType(file) } f.ContentType = mimeType - return client.Upload(f, "") + return client.Upload(f, "", toEncrypt) } } hash, err := doUpload() diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index 2bb601bdcb..f2ee999198 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -17,6 +17,7 @@ package main import ( + "fmt" "io" "io/ioutil" "net/http" @@ -29,6 +30,16 @@ import ( // TestCLISwarmUp tests that running 'swarm up' makes the resulting file // available from all nodes via the HTTP API func TestCLISwarmUp(t *testing.T) { + testCLISwarmUp(false, t) +} + +// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file +// available from all nodes via the HTTP API +func TestCLISwarmUpEncrypted(t *testing.T) { + testCLISwarmUp(true, t) +} + +func testCLISwarmUp(toEncrypt bool, t *testing.T) { log.Info("starting 3 node cluster") cluster := newTestCluster(t, 3) defer cluster.Shutdown() @@ -48,10 +59,16 @@ func TestCLISwarmUp(t *testing.T) { t.Fatal(err) } - // upload the file with 'swarm up' and expect a hash - log.Info("uploading file with 'swarm up'") - up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", tmp.Name()) - _, matches := up.ExpectRegexp(`[a-f\d]{64}`) + cmd := "up" + hashRegexp := `[a-f\d]{64}` + if toEncrypt { + cmd = "encrypted-up" + hashRegexp = `[a-f\d]{128}` + } + // upload the file with 'swarm up' or 'swarm encrypted-up' and expect a hash + log.Info(fmt.Sprintf("uploading file with '%s'", cmd)) + up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, cmd, tmp.Name()) + _, matches := up.ExpectRegexp(hashRegexp) up.ExpectExit() hash := matches[0] log.Info("file uploaded", "hash", hash) diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index 8165d52d7e..31ba3de510 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -125,11 +125,11 @@ func Open(path string) (*File, error) { // (if the manifest argument is non-empty) or creates a new manifest containing // the file, returning the resulting manifest hash (the file will then be // available at bzz://) -func (c *Client) Upload(file *File, manifest string) (string, error) { +func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, error) { if file.Size <= 0 { return "", errors.New("file size must be greater than zero") } - return c.TarUpload(manifest, &FileUploader{file}) + return c.TarUpload(manifest, &FileUploader{file}, toEncrypt) } // Download downloads a file with the given path from the swarm manifest with @@ -159,14 +159,14 @@ func (c *Client) Download(hash, path string) (*File, error) { // directory will then be available at bzz://path/to/file), with // the file specified in defaultPath being uploaded to the root of the manifest // (i.e. bzz://) -func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) { +func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bool) (string, error) { stat, err := os.Stat(dir) if err != nil { return "", err } else if !stat.IsDir() { return "", fmt.Errorf("not a directory: %s", dir) } - return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath}) + return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath}, toEncrypt) } // DownloadDirectory downloads the files contained in a swarm manifest under @@ -350,10 +350,19 @@ type UploadFn func(file *File) error // TarUpload uses the given Uploader to upload files to swarm as a tar stream, // returning the resulting manifest hash -func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) { +func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (string, error) { reqR, reqW := io.Pipe() defer reqR.Close() - req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR) + addr := hash + + // If there is a hash already (a manifest), then that manifest will determine if the upload has + // to be encrypted or not. If there is no manifest then the toEncrypt parameter decides if + // there is encryption or not. + if hash == "" && toEncrypt { + // This is the built-in address for the encrypted upload endpoint + addr = "encrypt" + } + req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+addr, reqR) if err != nil { return "", err } From c342a8c376513c2962999a86580ff83cf0c09133 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Tue, 10 Apr 2018 13:54:58 +0200 Subject: [PATCH 2/7] swarm/...: There is a response header to show if content was encrypted DPA.Retrieve() also returns this information --- swarm/api/api.go | 6 +++--- swarm/api/client/client_test.go | 6 +++--- swarm/api/filesystem.go | 2 +- swarm/api/http/server.go | 7 +++++-- swarm/api/http/server_test.go | 2 +- swarm/api/manifest.go | 8 ++++---- swarm/api/manifest_test.go | 6 +++--- swarm/fuse/fuse_file.go | 4 ++-- swarm/network/stream/common_test.go | 2 +- swarm/storage/dpa.go | 9 ++++++--- swarm/storage/dpa_test.go | 25 ++++++++++++++++++++----- 11 files changed, 49 insertions(+), 28 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index c7cb9c0662..8b48419e44 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -238,7 +238,7 @@ func (self *Api) Upload(uploadDir, index string, toEncrypt bool) (hash string, e } // DPA reader API -func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader { +func (self *Api) Retrieve(key storage.Key) (reader storage.LazySectionReader, isEncrypted bool) { return self.dpa.Retrieve(key) } @@ -344,7 +344,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe } else { mimeType = entry.ContentType log.Trace("content lookup key", "key", key, "mimetype", mimeType) - reader = self.dpa.Retrieve(key) + reader, _ = self.dpa.Retrieve(key) } } else { status = http.StatusNotFound @@ -482,7 +482,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte buf := make([]byte, buffSize) - oldReader := self.Retrieve(oldKey) + oldReader, _ := self.Retrieve(oldKey) io.ReadAtLeast(oldReader, buf, int(offset)) newReader := bytes.NewReader(content) diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index c1d144e370..fb053ad662 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -74,7 +74,7 @@ func TestClientUploadDownloadFiles(t *testing.T) { Size: int64(len(data)), }, } - hash, err := client.Upload(file, manifest) + hash, err := client.Upload(file, manifest, false) if err != nil { t.Fatal(err) } @@ -168,7 +168,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) { // upload the directory client := NewClient(srv.URL) defaultPath := filepath.Join(dir, testDirFiles[0]) - hash, err := client.UploadDirectory(dir, defaultPath, "") + hash, err := client.UploadDirectory(dir, defaultPath, "", false) if err != nil { t.Fatalf("error uploading directory: %s", err) } @@ -224,7 +224,7 @@ func TestClientFileList(t *testing.T) { defer os.RemoveAll(dir) client := NewClient(srv.URL) - hash, err := client.UploadDirectory(dir, "", "") + hash, err := client.UploadDirectory(dir, "", "", false) if err != nil { t.Fatalf("error uploading directory: %s", err) } diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index cd682e507d..55497d9498 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -273,7 +273,7 @@ func retrieveToFile(quitC chan bool, dpa *storage.DPA, key storage.Key, path str if err != nil { return err } - reader := dpa.Retrieve(key) + reader, _ := dpa.Retrieve(key) writer := bufio.NewWriter(f) size, err := reader.Size(quitC) if err != nil { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 73dd0ca13f..644b859484 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -556,13 +556,15 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { } // check the root chunk exists by retrieving the file's size - reader := s.api.Retrieve(key) + reader, isEncrypted := s.api.Retrieve(key) if _, err := reader.Size(nil); err != nil { getFail.Inc(1) Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", key, err), http.StatusNotFound) return } + w.Header().Set("X-Encrypted", fmt.Sprintf("%v", isEncrypted)) + switch { case r.uri.Raw() || r.uri.DeprecatedRaw(): // allow the request to overwrite the content type using a query @@ -619,11 +621,12 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { } // retrieve the entry's key and size - reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash))) + reader, isEncrypted := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash))) size, err := reader.Size(nil) if err != nil { return err } + w.Header().Set("X-Encrypted", fmt.Sprintf("%v", isEncrypted)) // write a tar header for the entry hdr := &tar.Header{ diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 901db68bc4..78a2c77c68 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -472,7 +472,7 @@ func TestBzzRootRedirect(t *testing.T) { Size: int64(len(data)), }, } - hash, err := client.Upload(file, "") + hash, err := client.Upload(file, "", false) if err != nil { t.Fatal(err) } diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 4b3d72909c..239ec8a776 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -205,12 +205,12 @@ type manifestTrieEntry struct { func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand log.Trace("manifest lookup", "key", hash) // retrieve manifest via DPA - manifestReader := dpa.Retrieve(hash) + manifestReader, isEncrypted := dpa.Retrieve(hash) log.Trace("reader retrieved", "key", hash) - return readManifest(manifestReader, hash, dpa, quitC) + return readManifest(manifestReader, hash, dpa, isEncrypted, quitC) } -func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, isEncrypted bool, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand // TODO check size for oversized manifests size, err := manifestReader.Size(quitC) @@ -245,7 +245,7 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp trie = &manifestTrie{ dpa: dpa, - encrypted: (len(hash) > dpa.HashSize()), + encrypted: isEncrypted, } for _, entry := range man.Entries { trie.addEntry(entry, quitC) diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index 27bf32007a..fb8f943924 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -44,7 +44,7 @@ func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...stri quitC := make(chan bool) dpa := storage.NewDPA(nil, storage.NewDPAParams()) ref := make([]byte, dpa.HashSize()) - trie, err := readManifest(manifest(paths...), ref, dpa, quitC) + trie, err := readManifest(manifest(paths...), ref, dpa, false, quitC) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } @@ -101,7 +101,7 @@ func TestExactMatch(t *testing.T) { mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map") dpa := storage.NewDPA(nil, storage.NewDPAParams()) ref := make([]byte, dpa.HashSize()) - trie, err := readManifest(mf, ref, dpa, quitC) + trie, err := readManifest(mf, ref, dpa, false, quitC) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } @@ -134,7 +134,7 @@ func TestAddFileWithManifestPath(t *testing.T) { } dpa := storage.NewDPA(nil, storage.NewDPAParams()) ref := make([]byte, dpa.HashSize()) - trie, err := readManifest(reader, ref, dpa, nil) + trie, err := readManifest(reader, ref, dpa, false, nil) if err != nil { t.Fatal(err) } diff --git a/swarm/fuse/fuse_file.go b/swarm/fuse/fuse_file.go index c94a0773f5..41a401eed5 100644 --- a/swarm/fuse/fuse_file.go +++ b/swarm/fuse/fuse_file.go @@ -82,7 +82,7 @@ func (file *SwarmFile) Attr(ctx context.Context, a *fuse.Attr) error { a.Gid = uint32(os.Getegid()) if file.fileSize == -1 { - reader := file.mountInfo.swarmApi.Retrieve(file.key) + reader, _ := file.mountInfo.swarmApi.Retrieve(file.key) quitC := make(chan bool) size, err := reader.Size(quitC) if err != nil { @@ -99,7 +99,7 @@ func (sf *SwarmFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse sf.lock.RLock() defer sf.lock.RUnlock() if sf.reader == nil { - sf.reader = sf.mountInfo.swarmApi.Retrieve(sf.key) + sf.reader, _ = sf.mountInfo.swarmApi.Retrieve(sf.key) } buf := make([]byte, req.Size) n, err := sf.reader.ReadAt(buf, req.Offset) diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 241bd7fc3b..8428fd3fb4 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -216,7 +216,7 @@ func (r *TestRegistry) APIs() []rpc.API { } func readAll(dpa *storage.DPA, hash []byte) (int64, error) { - r := dpa.Retrieve(hash) + r, _ := dpa.Retrieve(hash) buf := make([]byte, 1024) var n int var total int64 diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index af6e2a06ac..8070db8a8b 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -89,9 +89,12 @@ func NewDPA(store ChunkStore, params *DPAParams) *DPA { // FS-aware API and httpaccess // Chunk retrieval blocks on netStore requests with a timeout so reader will // report error if retrieval of chunks within requested range time out. -func (self *DPA) Retrieve(key Key) LazySectionReader { - getter := NewHasherStore(self.ChunkStore, self.hashFunc, len(key) > self.hashFunc().Size()) - return TreeJoin(key, getter, 0) +// It returns a reader with the chunk data and whether the content was encrypted +func (self *DPA) Retrieve(key Key) (reader LazySectionReader, isEncrypted bool) { + isEncrypted = len(key) > self.hashFunc().Size() + getter := NewHasherStore(self.ChunkStore, self.hashFunc, isEncrypted) + reader = TreeJoin(key, getter, 0) + return } // Public API. Main entry point for document storage directly. Used by the diff --git a/swarm/storage/dpa_test.go b/swarm/storage/dpa_test.go index 1126f05a52..a134347638 100644 --- a/swarm/storage/dpa_test.go +++ b/swarm/storage/dpa_test.go @@ -54,7 +54,10 @@ func testDpaRandom(toEncrypt bool, t *testing.T) { t.Errorf("Store error: %v", err) } wait() - resultReader := dpa.Retrieve(key) + resultReader, isEncrypted := dpa.Retrieve(key) + if isEncrypted != toEncrypt { + t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) + } resultSlice := make([]byte, len(slice)) n, err := resultReader.ReadAt(resultSlice, 0) if err != io.EOF { @@ -69,7 +72,10 @@ func testDpaRandom(toEncrypt bool, t *testing.T) { ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666) ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666) localStore.memStore = NewMemStore(db, defaultCacheCapacity) - resultReader = dpa.Retrieve(key) + resultReader, isEncrypted = dpa.Retrieve(key) + if isEncrypted != toEncrypt { + t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) + } for i := range resultSlice { resultSlice[i] = 0 } @@ -109,7 +115,10 @@ func testDPA_capacity(toEncrypt bool, t *testing.T) { t.Errorf("Store error: %v", err) } wait() - resultReader := dpa.Retrieve(key) + resultReader, isEncrypted := dpa.Retrieve(key) + if isEncrypted != toEncrypt { + t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) + } resultSlice := make([]byte, len(slice)) n, err := resultReader.ReadAt(resultSlice, 0) if err != io.EOF { @@ -125,14 +134,20 @@ func testDPA_capacity(toEncrypt bool, t *testing.T) { memStore.setCapacity(0) // check whether it is, indeed, empty dpa.ChunkStore = memStore - resultReader = dpa.Retrieve(key) + resultReader, isEncrypted = dpa.Retrieve(key) + if isEncrypted != toEncrypt { + t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) + } if _, err = resultReader.ReadAt(resultSlice, 0); err == nil { t.Errorf("Was able to read %d bytes from an empty memStore.", len(slice)) } // check how it works with localStore dpa.ChunkStore = localStore // localStore.dbStore.setCapacity(0) - resultReader = dpa.Retrieve(key) + resultReader, isEncrypted = dpa.Retrieve(key) + if isEncrypted != toEncrypt { + t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) + } for i := range resultSlice { resultSlice[i] = 0 } From b74f35b07862c7d7ddd6dd0c99036416bbaa8e03 Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Tue, 10 Apr 2018 14:09:05 +0200 Subject: [PATCH 3/7] swarm/api: Added encrypted test cases --- swarm/api/client/client_test.go | 20 ++++++++++++++++++-- swarm/api/http/server_test.go | 9 ++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index fb053ad662..dd5efabd1a 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -61,6 +61,14 @@ func TestClientUploadDownloadRaw(t *testing.T) { // TestClientUploadDownloadFiles test uploading and downloading files to swarm // manifests func TestClientUploadDownloadFiles(t *testing.T) { + testClientUploadDownloadFiles(false, t) +} + +func TestClientUploadDownloadFilesEncrypted(t *testing.T) { + testClientUploadDownloadFiles(true, t) +} + +func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() @@ -74,7 +82,7 @@ func TestClientUploadDownloadFiles(t *testing.T) { Size: int64(len(data)), }, } - hash, err := client.Upload(file, manifest, false) + hash, err := client.Upload(file, manifest, toEncrypt) if err != nil { t.Fatal(err) } @@ -217,6 +225,14 @@ func TestClientUploadDownloadDirectory(t *testing.T) { // TestClientFileList tests listing files in a swarm manifest func TestClientFileList(t *testing.T) { + testClientFileList(false, t) +} + +func TestClientFileListEncrypted(t *testing.T) { + testClientFileList(true, t) +} + +func testClientFileList(toEncrypt bool, t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() @@ -224,7 +240,7 @@ func TestClientFileList(t *testing.T) { defer os.RemoveAll(dir) client := NewClient(srv.URL) - hash, err := client.UploadDirectory(dir, "", "", false) + hash, err := client.UploadDirectory(dir, "", "", toEncrypt) if err != nil { t.Fatalf("error uploading directory: %s", err) } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 78a2c77c68..01ed796f8b 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -458,6 +458,13 @@ func testBzzGetPath(encrypted bool, t *testing.T) { // a trailing slash gets redirected to include the trailing slash so that // relative URLs work as expected. func TestBzzRootRedirect(t *testing.T) { + testBzzRootRedirect(false, t) +} +func TestBzzRootRedirectEncrypted(t *testing.T) { + testBzzRootRedirect(true, t) +} + +func testBzzRootRedirect(toEncrypt bool, t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() @@ -472,7 +479,7 @@ func TestBzzRootRedirect(t *testing.T) { Size: int64(len(data)), }, } - hash, err := client.Upload(file, "", false) + hash, err := client.Upload(file, "", toEncrypt) if err != nil { t.Fatal(err) } From f0dc791adf4f7a3b425384ab24984765ae0cad1d Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Wed, 11 Apr 2018 13:40:16 +0200 Subject: [PATCH 4/7] smd/swarm, swarm/api: client.go supports toEncrypt parameter --- cmd/swarm/manifest.go | 14 ++++++------- cmd/swarm/upload.go | 2 +- swarm/api/client/client.go | 37 ++++++++++++++++++++------------- swarm/api/client/client_test.go | 14 +++++++++++-- 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index 41a69a5d05..82166edf6c 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -131,13 +131,13 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin longestPathEntry = api.ManifestEntry{} ) - mroot, err := client.DownloadManifest(mhash) + mroot, isEncrypted, err := client.DownloadManifest(mhash) if err != nil { utils.Fatalf("Manifest download failed: %v", err) } //TODO: check if the "hash" to add is valid and present in swarm - _, err = client.DownloadManifest(hash) + _, _, err = client.DownloadManifest(hash) if err != nil { utils.Fatalf("Hash to add is not present: %v", err) } @@ -180,7 +180,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin mroot.Entries = append(mroot.Entries, newEntry) } - newManifestHash, err := client.UploadManifest(mroot) + newManifestHash, err := client.UploadManifest(mroot, isEncrypted) if err != nil { utils.Fatalf("Manifest upload failed: %v", err) } @@ -197,7 +197,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st longestPathEntry = api.ManifestEntry{} ) - mroot, err := client.DownloadManifest(mhash) + mroot, isEncrypted, err := client.DownloadManifest(mhash) if err != nil { utils.Fatalf("Manifest download failed: %v", err) } @@ -257,7 +257,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st mroot = newMRoot } - newManifestHash, err := client.UploadManifest(mroot) + newManifestHash, err := client.UploadManifest(mroot, isEncrypted) if err != nil { utils.Fatalf("Manifest upload failed: %v", err) } @@ -273,7 +273,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { longestPathEntry = api.ManifestEntry{} ) - mroot, err := client.DownloadManifest(mhash) + mroot, isEncrypted, err := client.DownloadManifest(mhash) if err != nil { utils.Fatalf("Manifest download failed: %v", err) } @@ -323,7 +323,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { mroot = newMRoot } - newManifestHash, err := client.UploadManifest(mroot) + newManifestHash, err := client.UploadManifest(mroot, isEncrypted) if err != nil { utils.Fatalf("Manifest upload failed: %v", err) } diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 058f93956d..bdcbc96581 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -84,7 +84,7 @@ func upload(ctx *cli.Context, toEncrypt bool) { utils.Fatalf("Error opening file: %s", err) } defer f.Close() - hash, err := client.UploadRaw(f, f.Size) + hash, err := client.UploadRaw(f, f.Size, toEncrypt) if err != nil { utils.Fatalf("Upload failed: %s", err) } diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index 31ba3de510..8b2edf4ff1 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -52,12 +52,17 @@ type Client struct { Gateway string } -// UploadRaw uploads raw data to swarm and returns the resulting hash -func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) { +// UploadRaw uploads raw data to swarm and returns the resulting hash. If toEncrypt is true it +// uploads encrypted data +func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, error) { if size <= 0 { return "", errors.New("data size must be greater than zero") } - req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/", r) + addr := "" + if toEncrypt { + addr = "encrypt" + } + req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/"+addr, r) if err != nil { return "", err } @@ -77,18 +82,20 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) { return string(data), nil } -// DownloadRaw downloads raw data from swarm -func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) { +// DownloadRaw downloads raw data from swarm and it returns a ReadCloser and a bool whether the +// content was encrypted +func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) { uri := c.Gateway + "/bzz-raw:/" + hash res, err := http.DefaultClient.Get(uri) if err != nil { - return nil, err + return nil, false, err } if res.StatusCode != http.StatusOK { res.Body.Close() - return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) + return nil, false, fmt.Errorf("unexpected HTTP status: %s", res.Status) } - return res.Body, nil + isEncrypted := (res.Header.Get("X-Encrypted") == "true") + return res.Body, isEncrypted, nil } // File represents a file in a swarm manifest and is used for uploading and @@ -229,26 +236,26 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error { } // UploadManifest uploads the given manifest to swarm -func (c *Client) UploadManifest(m *api.Manifest) (string, error) { +func (c *Client) UploadManifest(m *api.Manifest, toEncrypt bool) (string, error) { data, err := json.Marshal(m) if err != nil { return "", err } - return c.UploadRaw(bytes.NewReader(data), int64(len(data))) + return c.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt) } // DownloadManifest downloads a swarm manifest -func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) { - res, err := c.DownloadRaw(hash) +func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) { + res, isEncrypted, err := c.DownloadRaw(hash) if err != nil { - return nil, err + return nil, isEncrypted, err } defer res.Close() var manifest api.Manifest if err := json.NewDecoder(res).Decode(&manifest); err != nil { - return nil, err + return nil, isEncrypted, err } - return &manifest, nil + return &manifest, isEncrypted, nil } // List list files in a swarm manifest which have the given prefix, grouping diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index dd5efabd1a..347b3ddf27 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -31,6 +31,13 @@ import ( // TestClientUploadDownloadRaw test uploading and downloading raw data to swarm func TestClientUploadDownloadRaw(t *testing.T) { + testClientUploadDownloadRaw(false, t) +} +func TestClientUploadDownloadRawEncrypted(t *testing.T) { + testClientUploadDownloadRaw(true, t) +} + +func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() @@ -38,16 +45,19 @@ func TestClientUploadDownloadRaw(t *testing.T) { // upload some raw data data := []byte("foo123") - hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data))) + hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt) if err != nil { t.Fatal(err) } // check we can download the same data - res, err := client.DownloadRaw(hash) + res, isEncrypted, err := client.DownloadRaw(hash) if err != nil { t.Fatal(err) } + if isEncrypted != toEncrypt { + t.Fatalf("Expected encyption status %v got %v", toEncrypt, isEncrypted) + } defer res.Close() gotData, err := ioutil.ReadAll(res) if err != nil { From a0ae2a37dd8f064c6e4d60cc5e8a84ea027f234f Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 12 Apr 2018 14:44:15 +0200 Subject: [PATCH 5/7] cmd/swarm: Make encrypted upload a flag instead of a command --- cmd/swarm/main.go | 16 ++++++---------- cmd/swarm/upload.go | 11 ++--------- cmd/swarm/upload_test.go | 17 ++++++++++++----- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index f4ff0bb731..8ca1388447 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -141,6 +141,10 @@ var ( Name: "mime", Usage: "force mime type", } + SwarmEncryptedFlag = cli.BoolFlag{ + Name: "encrypted", + Usage: "use encrypted upload", + } SwarmPssEnabledFlag = cli.BoolFlag{ Name: "pss", Usage: "Enable pss (message passing over swarm)", @@ -218,19 +222,11 @@ The output of this command is supposed to be machine-readable. `, }, { - Action: nonEncryptedUpload, + Action: upload, Name: "up", Usage: "upload a file or directory to swarm using the HTTP API", ArgsUsage: " ", - Description: ` -"upload a file or directory to swarm using the HTTP API and prints the root hash", -`, - }, - { - Action: encryptedUpload, - Name: "encrypted-up", - Usage: "Upload a file or directory with encryption to swarm using the HTTP API. NOTE: Currently the reference for the uploaded content is non-deterministic, so you will receive different references if you upload it twice.", - ArgsUsage: " ", + Flags: []cli.Flag{SwarmEncryptedFlag}, Description: ` "upload a file or directory to swarm using the HTTP API and prints the root hash", `, diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index bdcbc96581..38207ddccb 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -35,15 +35,7 @@ import ( "gopkg.in/urfave/cli.v1" ) -func encryptedUpload(ctx *cli.Context) { - upload(ctx, true) -} - -func nonEncryptedUpload(ctx *cli.Context) { - upload(ctx, false) -} - -func upload(ctx *cli.Context, toEncrypt bool) { +func upload(ctx *cli.Context) { args := ctx.Args() var ( @@ -54,6 +46,7 @@ func upload(ctx *cli.Context, toEncrypt bool) { fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name) mimeType = ctx.GlobalString(SwarmUploadMimeType.Name) client = swarm.NewClient(bzzapi) + toEncrypt = ctx.Bool(SwarmEncryptedFlag.Name) file string ) diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index f2ee999198..a868a5540c 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -59,15 +59,22 @@ func testCLISwarmUp(toEncrypt bool, t *testing.T) { t.Fatal(err) } - cmd := "up" hashRegexp := `[a-f\d]{64}` + flags := []string{ + "--bzzapi", cluster.Nodes[0].URL, + "up", + tmp.Name()} if toEncrypt { - cmd = "encrypted-up" hashRegexp = `[a-f\d]{128}` + flags = []string{ + "--bzzapi", cluster.Nodes[0].URL, + "up", + "--encrypted", + tmp.Name()} } - // upload the file with 'swarm up' or 'swarm encrypted-up' and expect a hash - log.Info(fmt.Sprintf("uploading file with '%s'", cmd)) - up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, cmd, tmp.Name()) + // upload the file with 'swarm up' and expect a hash + log.Info(fmt.Sprintf("uploading file with 'swarm up'")) + up := runSwarm(t, flags...) _, matches := up.ExpectRegexp(hashRegexp) up.ExpectExit() hash := matches[0] From a565e892f4983e5f77ea744db89d5ce7f79bbd3c Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 12 Apr 2018 20:49:56 +0200 Subject: [PATCH 6/7] swarm/api: Rename X-Encrypted response header to X-Decrypted --- swarm/api/client/client.go | 2 +- swarm/api/http/server.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index 8b2edf4ff1..cc80624049 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -94,7 +94,7 @@ func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) { res.Body.Close() return nil, false, fmt.Errorf("unexpected HTTP status: %s", res.Status) } - isEncrypted := (res.Header.Get("X-Encrypted") == "true") + isEncrypted := (res.Header.Get("X-Decrypted") == "true") return res.Body, isEncrypted, nil } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 644b859484..9dc314a199 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -563,7 +563,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { return } - w.Header().Set("X-Encrypted", fmt.Sprintf("%v", isEncrypted)) + w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted)) switch { case r.uri.Raw() || r.uri.DeprecatedRaw(): @@ -626,7 +626,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { if err != nil { return err } - w.Header().Set("X-Encrypted", fmt.Sprintf("%v", isEncrypted)) + w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted)) // write a tar header for the entry hdr := &tar.Header{ From ea95a3fa0f7408e8d437d13755e63666f970319a Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 12 Apr 2018 21:00:18 +0200 Subject: [PATCH 7/7] swarm/api/http: Remove debug print --- swarm/api/http/server_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 01ed796f8b..74357d2c1e 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -257,7 +257,6 @@ func testBzzGetPath(encrypted bool, t *testing.T) { t.Fatal(err) } wait() - fmt.Println("!!!!!!!!!!", i, key[i]) } rootRef := key[2].Hex()