From 769160dd161e88a65ad8d9c90b2f76e0e7f00ea5 Mon Sep 17 00:00:00 2001 From: Zahoor Mohamed Date: Wed, 4 Jan 2017 03:48:09 +0530 Subject: [PATCH 1/4] Swarm: need tools to update/manipulate manifests #3498 --- cmd/swarm/main.go | 37 +++++++++++++++++++++++++++++++++++++ cmd/swarm/upload.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 04930760ee..392b049d57 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -152,6 +152,43 @@ The output of this command is supposed to be machine-readable. Prints the swarm hash of file or directory. `, }, + cli.Command{ + Name: "manifest", + Usage: "update a MANIFEST", + ArgsUsage: "manifest COMMAND", + Description: ` +Updates a MANIFEST by adding/removing/updating the hash of a path. +`, + Subcommands: []cli.Command{ + { + Action: add, + Name: "add", + Usage: "add a new path to the manifest", + ArgsUsage: " []", + Description: ` +Adds a new path to the manifest +`, + }, + { + Action: update, + Name: "update", + Usage: "update the hash for an already existing path in the manifest", + ArgsUsage: " []", + Description: ` +Update the hash for an already existing path in the manifest +`, + }, + { + Action: remove, + Name: "remove", + Usage: "removes a path from the manifest", + ArgsUsage: " ", + Description: ` +Removes a path from the manifest +`, + }, + }, + }, } app.Flags = []cli.Flag{ diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index d048bbc401..3b0c4f96e7 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -199,3 +199,31 @@ func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (strin content, err := ioutil.ReadAll(resp.Body) return string(content), err } + +func (c *client) downloadManifest(mhash string) (manifest, error) { + + mroot := manifest{} + req, err := http.NewRequest("GET", c.api + "/bzzr:/" + mhash, nil) + if err != nil { + return mroot, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return mroot, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + return mroot, fmt.Errorf("bad status: %s", resp.Status) + + } + content, err := ioutil.ReadAll(resp.Body) + + err = json.Unmarshal(content, &mroot) + if err != nil { + return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err) + } + return mroot, err +} + + From 18bf7ba0238f069162b62e945065e9e6e7ef2fad Mon Sep 17 00:00:00 2001 From: Zahoor Mohamed Date: Mon, 16 Jan 2017 13:21:51 +0530 Subject: [PATCH 2/4] adding manifest related command file --- cmd/swarm/manifest.go | 184 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 cmd/swarm/manifest.go diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go new file mode 100644 index 0000000000..383b0519f9 --- /dev/null +++ b/cmd/swarm/manifest.go @@ -0,0 +1,184 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// Command MANIFEST update +package main + +import ( + "encoding/json" + "fmt" + "gopkg.in/urfave/cli.v1" + "log" + "mime" + "path/filepath" + "strings" +) + +func add(ctx *cli.Context) { + + args := ctx.Args() + if len(args) < 3 { + log.Fatal("need atleast three arguments") + } + + var ( + mhash = args[0] + path = args[1] + hash = args[2] + + ) + + updateManifest (ctx, mhash, "add", path, hash) +} + +func update(ctx *cli.Context) { + + args := ctx.Args() + if len(args) < 3 { + log.Fatal("need atleast three arguments") + } + + var ( + mhash = args[0] + path = args[1] + hash = args[2] + + ) + + updateManifest (ctx, mhash, "update", path, hash) +} + +func remove(ctx *cli.Context) { + args := ctx.Args() + if len(args) < 2 { + log.Fatal("need atleast two arguments") + } + + var ( + mhash = args[0] + path = args[1] + + ) + + updateManifest (ctx, mhash, "remove", path, "") +} + + +func updateManifest(ctx *cli.Context, mhash , subcmd, path, hash string) { + + var ( + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) + client = &client{api: bzzapi} + mroot manifest + ) + + /* TODO: check for proper hash + if !common.IsHexAddress(mhash) { + log.Fatal(mhash, " is not a valid hash") + } + + if !common.IsHexAddress(hash) { + log.Fatal(hash, " is not a valid hash") + } + */ + + //mroot, err := client.downloadManifest(mhash) + //if err != nil { + // log.Fatalln("manifest download failed:", err) + //} + + switch subcmd { + + case "add": + for _, entry := range mroot.Entries { + if path == entry.Path { + log.Fatal(path, "Already present, not adding anything") + } + } + + newEntry := manifestEntry{ + Path: path, + Hash: hash, + ContentType: mime.TypeByExtension(filepath.Ext(path)), + } + mroot.Entries = append(mroot.Entries, newEntry) + break + + case "update": + foundEntry := bool(false) + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if path == entry.Path { + newEntry := manifestEntry{ + Path: entry.Path, + Hash: hash, + ContentType: entry.ContentType, + } + foundEntry = true + newMRoot.Entries = append(newMRoot.Entries, newEntry) + } else { + newMRoot.Entries = append(newMRoot.Entries, entry) + } + } + + if !foundEntry { + log.Fatal(path, " Path not present in the Manifest, not setting anything") + } + + mroot = newMRoot + break + + case "remove": + + foundEntry := bool(false) + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if path != entry.Path { + newEntry := manifestEntry{ + Path: entry.Path, + Hash: entry.Hash, + ContentType: entry.ContentType, + } + newMRoot.Entries = append(newMRoot.Entries, newEntry) + } else { + foundEntry = true + } + } + + if !foundEntry { + log.Fatal(path, "Path not present in the Manifest, not removing anything") + } + + mroot = newMRoot + break + + } + + if !wantManifest { + // Print the manifest. This is the only output to stdout. + mrootJSON, _ := json.MarshalIndent(mroot, "", " ") + fmt.Println(string(mrootJSON)) + return + } + + newManifestHash, err := client.uploadManifest(mroot) + if err != nil { + log.Fatalln("manifest upload failed:", err) + } + fmt.Println(newManifestHash) + +} From b4e9ea86a4520f4b52448dd81dec296f126c0b1b Mon Sep 17 00:00:00 2001 From: Zahoor Mohamed Date: Sat, 21 Jan 2017 19:06:11 +0530 Subject: [PATCH 3/4] Swarm: need tools to update/manipulate manifests #3498 Swarm: need tools to update/manipulate manifests #3498 --- cmd/swarm/manifest.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index 383b0519f9..e8a7a1d393 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -96,10 +96,10 @@ func updateManifest(ctx *cli.Context, mhash , subcmd, path, hash string) { } */ - //mroot, err := client.downloadManifest(mhash) - //if err != nil { - // log.Fatalln("manifest download failed:", err) - //} + mroot, err := client.downloadManifest(mhash) + if err != nil { + log.Fatalln("manifest download failed:", err) + } switch subcmd { From 82ddd30f22d0b55ff9751598d3da444dd3803ccf Mon Sep 17 00:00:00 2001 From: Zahoor Mohamed Date: Sun, 29 Jan 2017 02:19:02 +0530 Subject: [PATCH 4/4] Swarm: need tools to update/manipulate manifests #3498 --- cmd/swarm/manifest.go | 423 +++++++++++++++++++++++++++++------------- 1 file changed, 297 insertions(+), 126 deletions(-) diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index e8a7a1d393..094f7f960a 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -18,20 +18,20 @@ package main import ( - "encoding/json" - "fmt" "gopkg.in/urfave/cli.v1" "log" "mime" "path/filepath" "strings" + "fmt" + "encoding/json" ) func add(ctx *cli.Context) { args := ctx.Args() if len(args) < 3 { - log.Fatal("need atleast three arguments") + log.Fatal("need atleast three arguments []") } var ( @@ -39,134 +39,20 @@ func add(ctx *cli.Context) { path = args[1] hash = args[2] - ) - - updateManifest (ctx, mhash, "add", path, hash) -} - -func update(ctx *cli.Context) { - - args := ctx.Args() - if len(args) < 3 { - log.Fatal("need atleast three arguments") - } - - var ( - mhash = args[0] - path = args[1] - hash = args[2] - - ) - - updateManifest (ctx, mhash, "update", path, hash) -} - -func remove(ctx *cli.Context) { - args := ctx.Args() - if len(args) < 2 { - log.Fatal("need atleast two arguments") - } - - var ( - mhash = args[0] - path = args[1] - - ) - - updateManifest (ctx, mhash, "remove", path, "") -} - - -func updateManifest(ctx *cli.Context, mhash , subcmd, path, hash string) { - - var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + ctype string wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - client = &client{api: bzzapi} mroot manifest ) - /* TODO: check for proper hash - if !common.IsHexAddress(mhash) { - log.Fatal(mhash, " is not a valid hash") + + if len(args) > 3 { + ctype = args[3] + } else { + ctype = mime.TypeByExtension(filepath.Ext(path)) } - if !common.IsHexAddress(hash) { - log.Fatal(hash, " is not a valid hash") - } - */ - - mroot, err := client.downloadManifest(mhash) - if err != nil { - log.Fatalln("manifest download failed:", err) - } - - switch subcmd { - - case "add": - for _, entry := range mroot.Entries { - if path == entry.Path { - log.Fatal(path, "Already present, not adding anything") - } - } - - newEntry := manifestEntry{ - Path: path, - Hash: hash, - ContentType: mime.TypeByExtension(filepath.Ext(path)), - } - mroot.Entries = append(mroot.Entries, newEntry) - break - - case "update": - foundEntry := bool(false) - newMRoot := manifest{} - for _, entry := range mroot.Entries { - if path == entry.Path { - newEntry := manifestEntry{ - Path: entry.Path, - Hash: hash, - ContentType: entry.ContentType, - } - foundEntry = true - newMRoot.Entries = append(newMRoot.Entries, newEntry) - } else { - newMRoot.Entries = append(newMRoot.Entries, entry) - } - } - - if !foundEntry { - log.Fatal(path, " Path not present in the Manifest, not setting anything") - } - - mroot = newMRoot - break - - case "remove": - - foundEntry := bool(false) - newMRoot := manifest{} - for _, entry := range mroot.Entries { - if path != entry.Path { - newEntry := manifestEntry{ - Path: entry.Path, - Hash: entry.Hash, - ContentType: entry.ContentType, - } - newMRoot.Entries = append(newMRoot.Entries, newEntry) - } else { - foundEntry = true - } - } - - if !foundEntry { - log.Fatal(path, "Path not present in the Manifest, not removing anything") - } - - mroot = newMRoot - break - - } + newManifest := addEntryToManifest (ctx, mhash, path, hash, ctype) + fmt.Println(newManifest) if !wantManifest { // Print the manifest. This is the only output to stdout. @@ -174,11 +60,296 @@ func updateManifest(ctx *cli.Context, mhash , subcmd, path, hash string) { fmt.Println(string(mrootJSON)) return } +} + +func update(ctx *cli.Context) { + + args := ctx.Args() + if len(args) < 3 { + log.Fatal("need atleast three arguments ") + } + + var ( + mhash = args[0] + path = args[1] + hash = args[2] + + ctype string + wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) + mroot manifest + ) + if len(args) > 3 { + ctype = args[3] + } else { + ctype = mime.TypeByExtension(filepath.Ext(path)) + } + + newManifest := updateEntryInManifest (ctx, mhash, path, hash, ctype) + fmt.Println(newManifest) + + if !wantManifest { + // Print the manifest. This is the only output to stdout. + mrootJSON, _ := json.MarshalIndent(mroot, "", " ") + fmt.Println(string(mrootJSON)) + return + } +} + +func remove(ctx *cli.Context) { + args := ctx.Args() + if len(args) < 2 { + log.Fatal("need atleast two arguments ") + } + + var ( + mhash = args[0] + path = args[1] + + wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) + mroot manifest + ) + + newManifest := removeEntryFromManifest (ctx, mhash, path) + fmt.Println(newManifest) + + if !wantManifest { + // Print the manifest. This is the only output to stdout. + mrootJSON, _ := json.MarshalIndent(mroot, "", " ") + fmt.Println(string(mrootJSON)) + return + } +} + +func addEntryToManifest(ctx *cli.Context, mhash , path, hash , ctype string) string { + + var ( + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client = &client{api: bzzapi} + longestPathEntry = manifestEntry{ + Path: "", + Hash: "", + ContentType: "", + } + ) + + mroot, err := client.downloadManifest(mhash) + if err != nil { + log.Fatalln("manifest download failed:", err) + } + + //TODO: check if the "hash" to add is valid and present in swarm + + // See if we path is in this Manifest or do we have to dig deeper + for _, entry := range mroot.Entries { + if path == entry.Path { + log.Fatal(path, "Already present, not adding anything") + }else { + if entry.ContentType == "application/bzz-manifest+json" { + prfxlen := strings.HasPrefix(path, entry.Path) + if prfxlen && len(path) > len(longestPathEntry.Path) { + longestPathEntry = entry + } + } + } + } + + if longestPathEntry.Path != "" { + // Load the child Manifest add the entry there + newPath := path[len(longestPathEntry.Path):] + newHash := addEntryToManifest (ctx, longestPathEntry.Hash, newPath, hash, ctype) + + // Replace the hash for parent Manifests + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if longestPathEntry.Path == entry.Path { + entry.Hash = newHash + } + newMRoot.Entries = append(newMRoot.Entries, entry) + } + mroot = newMRoot + } else { + // Add the entry in the leaf Manifest + newEntry := manifestEntry{ + Path: path, + Hash: hash, + ContentType: ctype, + } + mroot.Entries = append(mroot.Entries, newEntry) + } + newManifestHash, err := client.uploadManifest(mroot) if err != nil { log.Fatalln("manifest upload failed:", err) } - fmt.Println(newManifestHash) + return newManifestHash + + } + +func updateEntryInManifest(ctx *cli.Context, mhash , path, hash , ctype string) string { + + var ( + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client = &client{api: bzzapi} + newEntry = manifestEntry{ + Path: "", + Hash: "", + ContentType: "", + } + longestPathEntry = manifestEntry{ + Path: "", + Hash: "", + ContentType: "", + } + ) + + mroot, err := client.downloadManifest(mhash) + if err != nil { + log.Fatalln("manifest download failed:", err) + } + + //TODO: check if the "hash" with which to update is valid and present in swarm + + + // See if we path is in this Manifest or do we have to dig deeper + for _, entry := range mroot.Entries { + if path == entry.Path { + newEntry = entry + }else { + if entry.ContentType == "application/bzz-manifest+json" { + prfxlen := strings.HasPrefix(path, entry.Path) + if prfxlen && len(path) > len(longestPathEntry.Path) { + longestPathEntry = entry + } + } + } + } + + if longestPathEntry.Path == "" && newEntry.Path == "" { + log.Fatal(path, " Path not present in the Manifest, not setting anything") + } + + if longestPathEntry.Path != "" { + // Load the child Manifest add the entry there + newPath := path[len(longestPathEntry.Path):] + newHash := addEntryToManifest (ctx, longestPathEntry.Hash, newPath, hash, ctype) + + // Replace the hash for parent Manifests + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if longestPathEntry.Path == entry.Path { + entry.Hash = newHash + } + newMRoot.Entries = append(newMRoot.Entries, entry) + + } + mroot = newMRoot + } + + if newEntry.Path != "" { + // Replace the hash for leaf Manifest + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if newEntry.Path == entry.Path { + myEntry := manifestEntry{ + Path: entry.Path, + Hash: hash, + ContentType: ctype, + } + newMRoot.Entries = append(newMRoot.Entries, myEntry) + } else { + newMRoot.Entries = append(newMRoot.Entries, entry) + } + } + mroot = newMRoot + } + + + newManifestHash, err := client.uploadManifest(mroot) + if err != nil { + log.Fatalln("manifest upload failed:", err) + } + return newManifestHash +} + +func removeEntryFromManifest(ctx *cli.Context, mhash , path string) string { + + var ( + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client = &client{api: bzzapi} + entryToRemove = manifestEntry{ + Path: "", + Hash: "", + ContentType: "", + } + longestPathEntry = manifestEntry{ + Path: "", + Hash: "", + ContentType: "", + } + ) + + mroot, err := client.downloadManifest(mhash) + if err != nil { + log.Fatalln("manifest download failed:", err) + } + + + + // See if we path is in this Manifest or do we have to dig deeper + for _, entry := range mroot.Entries { + if path == entry.Path { + entryToRemove = entry + }else { + if entry.ContentType == "application/bzz-manifest+json" { + prfxlen := strings.HasPrefix(path, entry.Path) + if prfxlen && len(path) > len(longestPathEntry.Path) { + longestPathEntry = entry + } + } + } + } + + if longestPathEntry.Path == "" && entryToRemove.Path == "" { + log.Fatal(path, "Path not present in the Manifest, not removing anything") + } + + if longestPathEntry.Path != "" { + // Load the child Manifest remove the entry there + newPath := path[len(longestPathEntry.Path):] + newHash := removeEntryFromManifest (ctx, longestPathEntry.Hash, newPath) + + // Replace the hash for parent Manifests + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if longestPathEntry.Path == entry.Path { + entry.Hash = newHash + } + newMRoot.Entries = append(newMRoot.Entries, entry) + } + mroot = newMRoot + } + + if entryToRemove.Path != "" { + // remove the entry in this Manifest + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if entryToRemove.Path != entry.Path { + newMRoot.Entries = append(newMRoot.Entries, entry) + } + } + mroot = newMRoot + } + + + newManifestHash, err := client.uploadManifest(mroot) + if err != nil { + log.Fatalln("manifest upload failed:", err) + } + return newManifestHash + + +} +