swarm: Add 'swarm ls' to list manifest directory trees

Ref #3540.

Signed-off-by: Lewis Marshall <lewis@lmars.net>

cmd/swarm/list.go

@@ -0,0 +1,58 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	"gopkg.in/urfave/cli.v1"
+)
+
+func list(ctx *cli.Context) {
+	args := ctx.Args()
+
+	if len(args) < 1 {
+		utils.Fatalf("Please supply a manifest reference as the first argument")
+	} else if len(args) > 2 {
+		utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
+	}
+	manifest := args[0]
+
+	var prefix string
+	if len(args) == 2 {
+		prefix = args[1]
+	}
+
+	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+	client := swarm.NewClient(bzzapi)
+	entries, err := client.ManifestFileList(manifest, prefix)
+	if err != nil {
+		utils.Fatalf("Failed to generate file and directory list: %s", err)
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
+	defer w.Flush()
+	fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
+	for _, entry := range entries {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
+	}
+}

cmd/swarm/main.go

@@ -145,6 +145,15 @@ The output of this command is supposed to be machine-readable.
 			ArgsUsage: " <file>",
 			Description: `
 "upload a file or directory to swarm using the HTTP API and prints the root hash",
+`,
+		},
+		{
+			Action:    list,
+			Name:      "ls",
+			Usage:     "list files and directories contained in a manifest",
+			ArgsUsage: " <manifest> [<prefix>]",
+			Description: `
+Lists files and directories contained in a manifest.
 `,
 		},
 		{

cmd/swarm/upload.go

@@ -68,7 +68,7 @@ func upload(ctx *cli.Context) {
 	if err != nil {
 		utils.Fatalf("Upload failed: %v", err)
 	}
-	mroot := swarm.Manifest{[]swarm.ManifestEntry{entry}}
+	mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
 	if !wantManifest {
 		// Print the manifest. This is the only output to stdout.
 		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")

swarm/api/client/client.go

@@ -186,3 +186,113 @@ func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
 	}
 	return mroot, err
 }
+
+// ManifestFileList downloads the manifest with the given hash and generates a
+// list of files and directory prefixes which have the specified prefix.
+//
+// For example, if the manifest represents the following directory structure:
+//
+// file1.txt
+// file2.txt
+// dir1/file3.txt
+// dir1/dir2/file4.txt
+//
+// Then:
+//
+// - a prefix of ""      would return [dir1/, file1.txt, file2.txt]
+// - a prefix of "file"  would return [file1.txt, file2.txt]
+// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
+func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
+	manifest, err := c.DownloadManifest(hash)
+	if err != nil {
+		return nil, err
+	}
+
+	// handleFile handles a manifest entry which is a direct reference to a
+	// file (i.e. it is not a swarm manifest)
+	handleFile := func(entry ManifestEntry) {
+		// ignore the file if it doesn't have the specified prefix
+		if !strings.HasPrefix(entry.Path, prefix) {
+			return
+		}
+		// if the path after the prefix contains a directory separator,
+		// add a directory prefix to the entries, otherwise add the
+		// file
+		suffix := strings.TrimPrefix(entry.Path, prefix)
+		if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
+			entries = append(entries, ManifestEntry{
+				Path:        prefix + suffix[:sepIndex+1],
+				ContentType: "DIR",
+			})
+		} else {
+			if entry.Path == "" {
+				entry.Path = "/"
+			}
+			entries = append(entries, entry)
+		}
+	}
+
+	// handleManifest handles a manifest entry which is a reference to
+	// another swarm manifest.
+	handleManifest := func(entry ManifestEntry) error {
+		// if the manifest's path is a prefix of the specified prefix
+		// then just recurse into the manifest by stripping its path
+		// from the prefix
+		if strings.HasPrefix(prefix, entry.Path) {
+			subPrefix := strings.TrimPrefix(prefix, entry.Path)
+			subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
+			if err != nil {
+				return err
+			}
+			// prefix the manifest's path to the sub entries and
+			// add them to the returned entries
+			for i, subEntry := range subEntries {
+				subEntry.Path = entry.Path + subEntry.Path
+				subEntries[i] = subEntry
+			}
+			entries = append(entries, subEntries...)
+			return nil
+		}
+
+		// if the manifest's path has the specified prefix, then if the
+		// path after the prefix contains a directory separator, add a
+		// directory prefix to the entries, otherwise recurse into the
+		// manifest
+		if strings.HasPrefix(entry.Path, prefix) {
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			sepIndex := strings.Index(suffix, "/")
+			if sepIndex > -1 {
+				entries = append(entries, ManifestEntry{
+					Path:        prefix + suffix[:sepIndex+1],
+					ContentType: "DIR",
+				})
+				return nil
+			}
+			subEntries, err := c.ManifestFileList(entry.Hash, "")
+			if err != nil {
+				return err
+			}
+			// prefix the manifest's path to the sub entries and
+			// add them to the returned entries
+			for i, subEntry := range subEntries {
+				subEntry.Path = entry.Path + subEntry.Path
+				subEntries[i] = subEntry
+			}
+			entries = append(entries, subEntries...)
+			return nil
+		}
+		return nil
+	}
+
+	for _, entry := range manifest.Entries {
+		if entry.ContentType == "application/bzz-manifest+json" {
+			if err := handleManifest(entry); err != nil {
+				return nil, err
+			}
+		} else {
+			handleFile(entry)
+		}
+	}
+
+	return
+}

swarm/api/client/client_test.go

@@ -0,0 +1,105 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/testutil"
+)
+
+func TestClientManifestFileList(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	dir, err := ioutil.TempDir("", "swarm-client-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	files := []string{
+		"file1.txt",
+		"file2.txt",
+		"dir1/file3.txt",
+		"dir1/file4.txt",
+		"dir2/file5.txt",
+		"dir2/dir3/file6.txt",
+		"dir2/dir4/file7.txt",
+		"dir2/dir4/file8.txt",
+	}
+	for _, file := range files {
+		path := filepath.Join(dir, file)
+		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+			t.Fatalf("error creating dir for %s: %s", path, err)
+		}
+		if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
+			t.Fatalf("error writing file %s: %s", path, err)
+		}
+	}
+
+	client := NewClient(srv.URL)
+
+	hash, err := client.UploadDirectory(dir, "")
+	if err != nil {
+		t.Fatalf("error uploading directory: %s", err)
+	}
+
+	ls := func(prefix string) []string {
+		entries, err := client.ManifestFileList(hash, prefix)
+		if err != nil {
+			t.Fatal(err)
+		}
+		paths := make([]string, len(entries))
+		for i, entry := range entries {
+			paths[i] = entry.Path
+		}
+		sort.Strings(paths)
+		return paths
+	}
+
+	tests := map[string][]string{
+		"":                    []string{"dir1/", "dir2/", "file1.txt", "file2.txt"},
+		"file":                []string{"file1.txt", "file2.txt"},
+		"file1":               []string{"file1.txt"},
+		"file2.txt":           []string{"file2.txt"},
+		"file12":              []string{},
+		"dir":                 []string{"dir1/", "dir2/"},
+		"dir1":                []string{"dir1/"},
+		"dir1/":               []string{"dir1/file3.txt", "dir1/file4.txt"},
+		"dir1/file":           []string{"dir1/file3.txt", "dir1/file4.txt"},
+		"dir1/file3.txt":      []string{"dir1/file3.txt"},
+		"dir1/file34":         []string{},
+		"dir2/":               []string{"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"},
+		"dir2/file":           []string{"dir2/file5.txt"},
+		"dir2/dir":            []string{"dir2/dir3/", "dir2/dir4/"},
+		"dir2/dir3/":          []string{"dir2/dir3/file6.txt"},
+		"dir2/dir4/":          []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
+		"dir2/dir4/file":      []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
+		"dir2/dir4/file7.txt": []string{"dir2/dir4/file7.txt"},
+		"dir2/dir4/file78":    []string{},
+	}
+	for prefix, expected := range tests {
+		actual := ls(prefix)
+		if !reflect.DeepEqual(actual, expected) {
+			t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
+		}
+	}
+}