From 8be14d0d1862f34ebe60e75706ada1a7e719efac Mon Sep 17 00:00:00 2001 From: Lewis Marshall Date: Fri, 3 Mar 2017 12:51:20 +0000 Subject: [PATCH] swarm: Add 'swarm ls' to list manifest directory trees Ref #3540. Signed-off-by: Lewis Marshall --- cmd/swarm/list.go | 58 +++++++++++++++++ cmd/swarm/main.go | 9 +++ cmd/swarm/upload.go | 2 +- swarm/api/client/client.go | 110 ++++++++++++++++++++++++++++++++ swarm/api/client/client_test.go | 105 ++++++++++++++++++++++++++++++ 5 files changed, 283 insertions(+), 1 deletion(-) create mode 100644 cmd/swarm/list.go create mode 100644 swarm/api/client/client_test.go diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go new file mode 100644 index 0000000000..3a68fef030 --- /dev/null +++ b/cmd/swarm/list.go @@ -0,0 +1,58 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "strings" + "text/tabwriter" + + "github.com/ethereum/go-ethereum/cmd/utils" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" + "gopkg.in/urfave/cli.v1" +) + +func list(ctx *cli.Context) { + args := ctx.Args() + + if len(args) < 1 { + utils.Fatalf("Please supply a manifest reference as the first argument") + } else if len(args) > 2 { + utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'") + } + manifest := args[0] + + var prefix string + if len(args) == 2 { + prefix = args[1] + } + + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) + entries, err := client.ManifestFileList(manifest, prefix) + if err != nil { + utils.Fatalf("Failed to generate file and directory list: %s", err) + } + + w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0) + defer w.Flush() + fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH") + for _, entry := range entries { + fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path) + } +} diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 1716771464..85960680fd 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -145,6 +145,15 @@ The output of this command is supposed to be machine-readable. ArgsUsage: " ", Description: ` "upload a file or directory to swarm using the HTTP API and prints the root hash", +`, + }, + { + Action: list, + Name: "ls", + Usage: "list files and directories contained in a manifest", + ArgsUsage: " []", + Description: ` +Lists files and directories contained in a manifest. `, }, { diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index d7168f79dd..696b907d22 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -68,7 +68,7 @@ func upload(ctx *cli.Context) { if err != nil { utils.Fatalf("Upload failed: %v", err) } - mroot := swarm.Manifest{[]swarm.ManifestEntry{entry}} + mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}} if !wantManifest { // Print the manifest. This is the only output to stdout. mrootJSON, _ := json.MarshalIndent(mroot, "", " ") diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index fda0c4801a..15e44f35d7 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -186,3 +186,113 @@ func (c *Client) DownloadManifest(mhash string) (Manifest, error) { } return mroot, err } + +// ManifestFileList downloads the manifest with the given hash and generates a +// list of files and directory prefixes which have the specified prefix. +// +// For example, if the manifest represents the following directory structure: +// +// file1.txt +// file2.txt +// dir1/file3.txt +// dir1/dir2/file4.txt +// +// Then: +// +// - a prefix of "" would return [dir1/, file1.txt, file2.txt] +// - a prefix of "file" would return [file1.txt, file2.txt] +// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt] +func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) { + manifest, err := c.DownloadManifest(hash) + if err != nil { + return nil, err + } + + // handleFile handles a manifest entry which is a direct reference to a + // file (i.e. it is not a swarm manifest) + handleFile := func(entry ManifestEntry) { + // ignore the file if it doesn't have the specified prefix + if !strings.HasPrefix(entry.Path, prefix) { + return + } + // if the path after the prefix contains a directory separator, + // add a directory prefix to the entries, otherwise add the + // file + suffix := strings.TrimPrefix(entry.Path, prefix) + if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 { + entries = append(entries, ManifestEntry{ + Path: prefix + suffix[:sepIndex+1], + ContentType: "DIR", + }) + } else { + if entry.Path == "" { + entry.Path = "/" + } + entries = append(entries, entry) + } + } + + // handleManifest handles a manifest entry which is a reference to + // another swarm manifest. + handleManifest := func(entry ManifestEntry) error { + // if the manifest's path is a prefix of the specified prefix + // then just recurse into the manifest by stripping its path + // from the prefix + if strings.HasPrefix(prefix, entry.Path) { + subPrefix := strings.TrimPrefix(prefix, entry.Path) + subEntries, err := c.ManifestFileList(entry.Hash, subPrefix) + if err != nil { + return err + } + // prefix the manifest's path to the sub entries and + // add them to the returned entries + for i, subEntry := range subEntries { + subEntry.Path = entry.Path + subEntry.Path + subEntries[i] = subEntry + } + entries = append(entries, subEntries...) + return nil + } + + // if the manifest's path has the specified prefix, then if the + // path after the prefix contains a directory separator, add a + // directory prefix to the entries, otherwise recurse into the + // manifest + if strings.HasPrefix(entry.Path, prefix) { + suffix := strings.TrimPrefix(entry.Path, prefix) + sepIndex := strings.Index(suffix, "/") + if sepIndex > -1 { + entries = append(entries, ManifestEntry{ + Path: prefix + suffix[:sepIndex+1], + ContentType: "DIR", + }) + return nil + } + subEntries, err := c.ManifestFileList(entry.Hash, "") + if err != nil { + return err + } + // prefix the manifest's path to the sub entries and + // add them to the returned entries + for i, subEntry := range subEntries { + subEntry.Path = entry.Path + subEntry.Path + subEntries[i] = subEntry + } + entries = append(entries, subEntries...) + return nil + } + return nil + } + + for _, entry := range manifest.Entries { + if entry.ContentType == "application/bzz-manifest+json" { + if err := handleManifest(entry); err != nil { + return nil, err + } + } else { + handleFile(entry) + } + } + + return +} diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go new file mode 100644 index 0000000000..1354744750 --- /dev/null +++ b/swarm/api/client/client_test.go @@ -0,0 +1,105 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package client + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/swarm/testutil" +) + +func TestClientManifestFileList(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + dir, err := ioutil.TempDir("", "swarm-client-test") + if err != nil { + t.Fatal(err) + } + files := []string{ + "file1.txt", + "file2.txt", + "dir1/file3.txt", + "dir1/file4.txt", + "dir2/file5.txt", + "dir2/dir3/file6.txt", + "dir2/dir4/file7.txt", + "dir2/dir4/file8.txt", + } + for _, file := range files { + path := filepath.Join(dir, file) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatalf("error creating dir for %s: %s", path, err) + } + if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil { + t.Fatalf("error writing file %s: %s", path, err) + } + } + + client := NewClient(srv.URL) + + hash, err := client.UploadDirectory(dir, "") + if err != nil { + t.Fatalf("error uploading directory: %s", err) + } + + ls := func(prefix string) []string { + entries, err := client.ManifestFileList(hash, prefix) + if err != nil { + t.Fatal(err) + } + paths := make([]string, len(entries)) + for i, entry := range entries { + paths[i] = entry.Path + } + sort.Strings(paths) + return paths + } + + tests := map[string][]string{ + "": []string{"dir1/", "dir2/", "file1.txt", "file2.txt"}, + "file": []string{"file1.txt", "file2.txt"}, + "file1": []string{"file1.txt"}, + "file2.txt": []string{"file2.txt"}, + "file12": []string{}, + "dir": []string{"dir1/", "dir2/"}, + "dir1": []string{"dir1/"}, + "dir1/": []string{"dir1/file3.txt", "dir1/file4.txt"}, + "dir1/file": []string{"dir1/file3.txt", "dir1/file4.txt"}, + "dir1/file3.txt": []string{"dir1/file3.txt"}, + "dir1/file34": []string{}, + "dir2/": []string{"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"}, + "dir2/file": []string{"dir2/file5.txt"}, + "dir2/dir": []string{"dir2/dir3/", "dir2/dir4/"}, + "dir2/dir3/": []string{"dir2/dir3/file6.txt"}, + "dir2/dir4/": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, + "dir2/dir4/file": []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, + "dir2/dir4/file7.txt": []string{"dir2/dir4/file7.txt"}, + "dir2/dir4/file78": []string{}, + } + for prefix, expected := range tests { + actual := ls(prefix) + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual) + } + } +}