forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-20 10:52:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

swarm: remove swarm (use ipfs)

Browse source
This commit is contained in:
Luke Williams 2019-11-26 18:06:59 +01:00
parent 4753ebd341
commit aaec135b34
317 changed files with 9 additions and 73081 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
Makefile
View file

@ -2,7 +2,7 @@
# with Go source code. If you know what GOPATH is then you probably
# don't need to bother with make.
.PHONY: gubiq android ios gubiq-cross swarm evm all test clean
.PHONY: gubiq android ios gubiq-cross evm all test clean
.PHONY: gubiq-linux gubiq-linux-386 gubiq-linux-amd64 gubiq-linux-mips64 gubiq-linux-mips64le
.PHONY: gubiq-linux-arm gubiq-linux-arm-5 gubiq-linux-arm-6 gubiq-linux-arm-7 gubiq-linux-arm64
.PHONY: gubiq-darwin gubiq-darwin-386 gubiq-darwin-amd64
@ -16,11 +16,6 @@ gubiq:
@echo "Done building."
@echo "Run \"$(GOBIN)/gubiq\" to launch gubiq."
swarm:
build/env.sh go run build/ci.go install ./cmd/swarm
@echo "Done building."
@echo "Run \"$(GOBIN)/swarm\" to launch swarm."
all:
build/env.sh go run build/ci.go install
@ -57,9 +52,6 @@ devtools:
@type "solc" 2> /dev/null || echo 'Please install solc'
@type "protoc" 2> /dev/null || echo 'Please install protoc'
swarm-devtools:
env GOBIN= go install ./cmd/swarm/mimegen
# Cross Compilation Targets (xgo)
gubiq-cross: gubiq-linux gubiq-darwin gubiq-windows gubiq-android gubiq-ios

40
build/ci.go
View file

@ -60,7 +60,6 @@ import (
"github.com/ubiq/go-ubiq/internal/build"
"github.com/ubiq/go-ubiq/params"
sv "github.com/ubiq/go-ubiq/swarm/version"
)
var (
@ -82,12 +81,6 @@ var (
executablePath("wnode"),
}
// Files that end up in the swarm*.zip archive.
swarmArchiveFiles = []string{
"COPYING",
executablePath("swarm"),
}
// A debian package is created for all executables listed here.
debExecutables = []debExecutable{
{
@ -116,16 +109,7 @@ var (
},
{
BinaryName: "wnode",
Description: "Ethereum Whisper diagnostic tool",
},
}
// A debian package is created for all executables listed here.
debSwarmExecutables = []debExecutable{
{
BinaryName: "swarm",
PackageName: "ethereum-swarm",
Description: "Ethereum Swarm daemon and tools",
Description: "Ubiq Whisper diagnostic tool",
},
}
@ -135,21 +119,11 @@ var (
Executables: debExecutables,
}
debSwarm = debPackage{
Name: "ethereum-swarm",
Version: sv.Version,
Executables: debSwarmExecutables,
}
// Debian meta packages to build and push to Ubuntu PPA
debPackages = []debPackage{
debSwarm,
debEthereum,
}
// Packages to be cross-compiled by the xgo command
allCrossCompiledArchiveFiles = append(allToolsArchiveFiles, swarmArchiveFiles...)
// Distros for which packages are created.
// Note: vivid is unsupported because there is no golang-1.6 package for it.
// Note: wily is unsupported because it was officially deprecated on lanchpad.
@ -402,10 +376,7 @@ func doArchive(cmdline []string) {
basegubiq = archiveBasename(*arch, params.ArchiveVersion(env.Commit))
gubiq = "gubiq-" + basegubiq + ext
alltools = "gubiq-alltools-" + basegubiq + ext
baseswarm = archiveBasename(*arch, sv.ArchiveVersion(env.Commit))
swarm = "swarm-" + baseswarm + ext
alltools = "gubiq-alltools-" + basegubiq + ext
)
maybeSkipArchive(env)
if err := build.WriteArchive(gubiq, gubiqArchiveFiles); err != nil {
@ -414,10 +385,7 @@ func doArchive(cmdline []string) {
if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil {
log.Fatal(err)
}
if err := build.WriteArchive(swarm, swarmArchiveFiles); err != nil {
log.Fatal(err)
}
for _, archive := range []string{gubiq, alltools, swarm} {
for _, archive := range []string{gubiq, alltools} {
if err := archiveUpload(archive, *upload, *signer); err != nil {
log.Fatal(err)
}
@ -1021,7 +989,7 @@ func doXgo(cmdline []string) {
if *alltools {
args = append(args, []string{"--dest", GOBIN}...)
for _, res := range allCrossCompiledArchiveFiles {
for _, res := range allToolsArchiveFiles {
if strings.HasPrefix(res, GOBIN) {
// Binary tool found, cross build it explicitly
args = append(args, "./"+filepath.Join("cmd", filepath.Base(res)))

2
build/update-license.go
View file

@ -72,8 +72,6 @@ var (
"internal/jsre/deps",
"log/",
"common/bitutil/bitutil",
// don't license generated files
"contracts/chequebook/contract/code.go",
}
// paths with this prefix are licensed as GPL. all other files are LGPL.

297
cmd/swarm/access.go
View file

@ -1,297 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"crypto/rand"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"strings"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/swarm/api"
"github.com/ubiq/go-ubiq/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
var (
salt = make([]byte, 32)
accessCommand = cli.Command{
CustomHelpTemplate: helpTemplate,
Name: "access",
Usage: "encrypts a reference and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root manifest",
Subcommands: []cli.Command{
{
CustomHelpTemplate: helpTemplate,
Name: "new",
Usage: "encrypts a reference and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
Subcommands: []cli.Command{
{
Action: accessNewPass,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
utils.PasswordFileFlag,
SwarmDryRunFlag,
},
Name: "pass",
Usage: "encrypts a reference with a password and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
{
Action: accessNewPK,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
utils.PasswordFileFlag,
SwarmDryRunFlag,
SwarmAccessGrantKeyFlag,
},
Name: "pk",
Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
{
Action: accessNewACT,
CustomHelpTemplate: helpTemplate,
Flags: []cli.Flag{
SwarmAccessGrantKeysFlag,
SwarmDryRunFlag,
utils.PasswordFileFlag,
},
Name: "act",
Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest",
ArgsUsage: "<ref>",
Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest",
},
},
},
},
}
)
func init() {
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
panic("reading from crypto/rand failed: " + err.Error())
}
}
func accessNewPass(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("Expected 1 argument - the ref")
}
var (
ae *api.AccessEntry
accessKey []byte
err error
ref = args[0]
password = getPassPhrase("", 0, makePasswordList(ctx))
dryRun = ctx.Bool(SwarmDryRunFlag.Name)
)
accessKey, ae, err = api.DoPassword(ctx, password, salt)
if err != nil {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
if err != nil {
utils.Fatalf("had an error generating the manifest: %v", err)
}
if dryRun {
err = printManifests(m, nil)
if err != nil {
utils.Fatalf("had an error printing the manifests: %v", err)
}
} else {
err = uploadManifests(ctx, m, nil)
if err != nil {
utils.Fatalf("had an error uploading the manifests: %v", err)
}
}
}
func accessNewPK(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("Expected 1 argument - the ref")
}
var (
ae *api.AccessEntry
sessionKey []byte
err error
ref = args[0]
privateKey = getPrivKey(ctx)
granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name)
dryRun = ctx.Bool(SwarmDryRunFlag.Name)
)
sessionKey, ae, err = api.DoPK(ctx, privateKey, granteePublicKey, salt)
if err != nil {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae)
if err != nil {
utils.Fatalf("had an error generating the manifest: %v", err)
}
if dryRun {
err = printManifests(m, nil)
if err != nil {
utils.Fatalf("had an error printing the manifests: %v", err)
}
} else {
err = uploadManifests(ctx, m, nil)
if err != nil {
utils.Fatalf("had an error uploading the manifests: %v", err)
}
}
}
func accessNewACT(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 1 {
utils.Fatalf("Expected 1 argument - the ref")
}
var (
ae *api.AccessEntry
actManifest *api.Manifest
accessKey []byte
err error
ref = args[0]
pkGrantees = []string{}
passGrantees = []string{}
pkGranteesFilename = ctx.String(SwarmAccessGrantKeysFlag.Name)
passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name)
privateKey = getPrivKey(ctx)
dryRun = ctx.Bool(SwarmDryRunFlag.Name)
)
if pkGranteesFilename == "" && passGranteesFilename == "" {
utils.Fatalf("you have to provide either a grantee public-keys file or an encryption passwords file (or both)")
}
if pkGranteesFilename != "" {
bytes, err := ioutil.ReadFile(pkGranteesFilename)
if err != nil {
utils.Fatalf("had an error reading the grantee public key list")
}
pkGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n")
}
if passGranteesFilename != "" {
bytes, err := ioutil.ReadFile(passGranteesFilename)
if err != nil {
utils.Fatalf("could not read password filename: %v", err)
}
passGrantees = strings.Split(strings.Trim(string(bytes), "\n"), "\n")
}
accessKey, ae, actManifest, err = api.DoACT(ctx, privateKey, salt, pkGrantees, passGrantees)
if err != nil {
utils.Fatalf("error generating ACT manifest: %v", err)
}
if err != nil {
utils.Fatalf("error getting session key: %v", err)
}
m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae)
if err != nil {
utils.Fatalf("error generating root access manifest: %v", err)
}
if dryRun {
err = printManifests(m, actManifest)
if err != nil {
utils.Fatalf("had an error printing the manifests: %v", err)
}
} else {
err = uploadManifests(ctx, m, actManifest)
if err != nil {
utils.Fatalf("had an error uploading the manifests: %v", err)
}
}
}
func printManifests(rootAccessManifest, actManifest *api.Manifest) error {
js, err := json.Marshal(rootAccessManifest)
if err != nil {
return err
}
fmt.Println(string(js))
if actManifest != nil {
js, err := json.Marshal(actManifest)
if err != nil {
return err
}
fmt.Println(string(js))
}
return nil
}
func uploadManifests(ctx *cli.Context, rootAccessManifest, actManifest *api.Manifest) error {
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := client.NewClient(bzzapi)
var (
key string
err error
)
if actManifest != nil {
key, err = client.UploadManifest(actManifest, false)
if err != nil {
return err
}
rootAccessManifest.Entries[0].Access.Act = key
}
key, err = client.UploadManifest(rootAccessManifest, false)
if err != nil {
return err
}
fmt.Println(key)
return nil
}
// makePasswordList reads password lines from the file specified by the global --password flag
// and also by the same subcommand --password flag.
// This function ia a fork of utils.MakePasswordList to lookup cli context for subcommand.
// Function ctx.SetGlobal is not setting the global flag value that can be accessed
// by ctx.GlobalString using the current version of cli package.
func makePasswordList(ctx *cli.Context) []string {
path := ctx.GlobalString(utils.PasswordFileFlag.Name)
if path == "" {
path = ctx.String(utils.PasswordFileFlag.Name)
if path == "" {
return nil
}
}
text, err := ioutil.ReadFile(path)
if err != nil {
utils.Fatalf("Failed to read password file: %v", err)
}
lines := strings.Split(string(text), "\n")
// Sanitise DOS line endings.
for i := range lines {
lines[i] = strings.TrimRight(lines[i], "\r")
}
return lines
}

614
cmd/swarm/access_test.go
View file

@ -1,614 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"crypto/rand"
"encoding/hex"
"encoding/json"
"io"
"io/ioutil"
gorand "math/rand"
"net/http"
"os"
"runtime"
"strings"
"testing"
"time"
"github.com/ubiq/go-ubiq/crypto"
"github.com/ubiq/go-ubiq/crypto/ecies"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/swarm/api"
swarmapi "github.com/ubiq/go-ubiq/swarm/api/client"
"github.com/ubiq/go-ubiq/swarm/testutil"
"golang.org/x/crypto/sha3"
)
const (
hashRegexp = `[a-f\d]{128}`
data = "notsorandomdata"
)
var DefaultCurve = crypto.S256()
func TestACT(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
initCluster(t)
cases := []struct {
name string
f func(t *testing.T)
}{
{"Password", testPassword},
{"PK", testPK},
{"ACTWithoutBogus", testACTWithoutBogus},
{"ACTWithBogus", testACTWithBogus},
}
for _, tc := range cases {
t.Run(tc.name, tc.f)
}
}
// testPassword tests for the correct creation of an ACT manifest protected by a password.
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded
// is then fetched through 2nd node. since the tested code is not key-aware - we can just
// fetch from the 2nd node using HTTP BasicAuth
func testPassword(t *testing.T) {
dataFilename := testutil.TempFileWithContent(t, data)
defer os.RemoveAll(dataFilename)
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
"--encrypt",
dataFilename)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
ref := matches[0]
tmp, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
password := "smth"
passwordFilename := testutil.TempFileWithContent(t, "smth")
defer os.RemoveAll(passwordFilename)
up = runSwarm(t,
"access",
"new",
"pass",
"--dry-run",
"--password",
passwordFilename,
ref,
)
_, matches = up.ExpectRegexp(".+")
up.ExpectExit()
if len(matches) == 0 {
t.Fatalf("stdout not matched")
}
var m api.Manifest
err = json.Unmarshal([]byte(matches[0]), &m)
if err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if len(m.Entries) != 1 {
t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
}
e := m.Entries[0]
ct := "application/bzz-manifest+json"
if e.ContentType != ct {
t.Errorf("expected %q content type, got %q", ct, e.ContentType)
}
if e.Access == nil {
t.Fatal("manifest access is nil")
}
a := e.Access
if a.Type != "pass" {
t.Errorf(`got access type %q, expected "pass"`, a.Type)
}
if len(a.Salt) < 32 {
t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
}
if a.KdfParams == nil {
t.Fatal("manifest access kdf params is nil")
}
if a.Publisher != "" {
t.Fatal("should be empty")
}
client := swarmapi.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false)
if err != nil {
t.Fatal(err)
}
url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
httpClient := &http.Client{}
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusUnauthorized {
t.Fatal("should be a 401")
}
authHeader := response.Header.Get("WWW-Authenticate")
if authHeader == "" {
t.Fatal("should be something here")
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
t.Fatal(err)
}
req.SetBasicAuth("", password)
response, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
t.Errorf("expected status %v, got %v", http.StatusOK, response.StatusCode)
}
d, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatal(err)
}
if string(d) != data {
t.Errorf("expected decrypted data %q, got %q", data, string(d))
}
wrongPasswordFilename := testutil.TempFileWithContent(t, "just wr0ng")
defer os.RemoveAll(wrongPasswordFilename)
//download file with 'swarm down' with wrong password
up = runSwarm(t,
"--bzzapi",
cluster.Nodes[0].URL,
"down",
"bzz:/"+hash,
tmp,
"--password",
wrongPasswordFilename)
_, matches = up.ExpectRegexp("unauthorized")
if len(matches) != 1 && matches[0] != "unauthorized" {
t.Fatal(`"unauthorized" not found in output"`)
}
up.ExpectExit()
}
// testPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee).
// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry
// The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears.
// Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware,
// the test will fail if the proxy's given private key is not granted on the ACT.
func testPK(t *testing.T) {
dataFilename := testutil.TempFileWithContent(t, data)
defer os.RemoveAll(dataFilename)
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
"--encrypt",
dataFilename)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
ref := matches[0]
pk := cluster.Nodes[0].PrivateKey
granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
if err != nil {
t.Fatal(err)
}
passwordFilename := testutil.TempFileWithContent(t, testPassphrase)
defer os.RemoveAll(passwordFilename)
_, publisherAccount := getTestAccount(t, publisherDir)
up = runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passwordFilename,
"--datadir",
publisherDir,
"--bzzapi",
cluster.Nodes[0].URL,
"access",
"new",
"pk",
"--dry-run",
"--grant-key",
hex.EncodeToString(granteePubKey),
ref,
)
_, matches = up.ExpectRegexp(".+")
up.ExpectExit()
if len(matches) == 0 {
t.Fatalf("stdout not matched")
}
//get the public key from the publisher directory
publicKeyFromDataDir := runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passwordFilename,
"--datadir",
publisherDir,
"print-keys",
"--compressed",
)
_, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+")
publicKeyFromDataDir.ExpectExit()
pkComp := strings.Split(publicKeyString[0], "=")[1]
var m api.Manifest
err = json.Unmarshal([]byte(matches[0]), &m)
if err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if len(m.Entries) != 1 {
t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
}
e := m.Entries[0]
ct := "application/bzz-manifest+json"
if e.ContentType != ct {
t.Errorf("expected %q content type, got %q", ct, e.ContentType)
}
if e.Access == nil {
t.Fatal("manifest access is nil")
}
a := e.Access
if a.Type != "pk" {
t.Errorf(`got access type %q, expected "pk"`, a.Type)
}
if len(a.Salt) < 32 {
t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
}
if a.KdfParams != nil {
t.Fatal("manifest access kdf params should be nil")
}
if a.Publisher != pkComp {
t.Fatal("publisher key did not match")
}
client := swarmapi.NewClient(cluster.Nodes[0].URL)
hash, err := client.UploadManifest(&m, false)
if err != nil {
t.Fatal(err)
}
httpClient := &http.Client{}
url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusOK {
t.Fatal("should be a 200")
}
d, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatal(err)
}
if string(d) != data {
t.Errorf("expected decrypted data %q, got %q", data, string(d))
}
}
// testACTWithoutBogus tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized)
func testACTWithoutBogus(t *testing.T) {
testACT(t, 0)
}
// testACTWithBogus tests the creation of the ACT manifest end-to-end, with 100 bogus entries (i.e. 100 EC keys + default scenario = 3 nodes 1 unauthorized = 103 keys in the ACT manifest)
func testACTWithBogus(t *testing.T) {
testACT(t, 100)
}
// testACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection
// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data
// set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access.
// the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails.
// the publisher uploads through one of the nodes then disappears.
func testACT(t *testing.T, bogusEntries int) {
var uploadThroughNode = cluster.Nodes[0]
client := swarmapi.NewClient(uploadThroughNode.URL)
r1 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`)
dataFilename := testutil.TempFileWithContent(t, data)
defer os.RemoveAll(dataFilename)
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t,
"--bzzapi",
cluster.Nodes[0].URL,
"up",
"--encrypt",
dataFilename)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
ref := matches[0]
grantees := []string{}
for i, v := range cluster.Nodes {
if i == nodeToSkip {
continue
}
pk := v.PrivateKey
granteePubKey := crypto.CompressPubkey(&pk.PublicKey)
grantees = append(grantees, hex.EncodeToString(granteePubKey))
}
if bogusEntries > 0 {
bogusGrantees := []string{}
for i := 0; i < bogusEntries; i++ {
prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil)
if err != nil {
t.Fatal(err)
}
bogusGrantees = append(bogusGrantees, hex.EncodeToString(crypto.CompressPubkey(&prv.ExportECDSA().PublicKey)))
}
r2 := gorand.New(gorand.NewSource(time.Now().UnixNano()))
for i := 0; i < len(grantees); i++ {
insertAtIdx := r2.Intn(len(bogusGrantees))
bogusGrantees = append(bogusGrantees[:insertAtIdx], append([]string{grantees[i]}, bogusGrantees[insertAtIdx:]...)...)
}
grantees = bogusGrantees
}
granteesPubkeyListFile := testutil.TempFileWithContent(t, strings.Join(grantees, "\n"))
defer os.RemoveAll(granteesPubkeyListFile)
publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(publisherDir)
passwordFilename := testutil.TempFileWithContent(t, testPassphrase)
defer os.RemoveAll(passwordFilename)
actPasswordFilename := testutil.TempFileWithContent(t, "smth")
defer os.RemoveAll(actPasswordFilename)
_, publisherAccount := getTestAccount(t, publisherDir)
up = runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passwordFilename,
"--datadir",
publisherDir,
"--bzzapi",
cluster.Nodes[0].URL,
"access",
"new",
"act",
"--grant-keys",
granteesPubkeyListFile,
"--password",
actPasswordFilename,
ref,
)
_, matches = up.ExpectRegexp(`[a-f\d]{64}`)
up.ExpectExit()
if len(matches) == 0 {
t.Fatalf("stdout not matched")
}
//get the public key from the publisher directory
publicKeyFromDataDir := runSwarm(t,
"--bzzaccount",
publisherAccount.Address.String(),
"--password",
passwordFilename,
"--datadir",
publisherDir,
"print-keys",
"--compressed",
)
_, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+")
publicKeyFromDataDir.ExpectExit()
pkComp := strings.Split(publicKeyString[0], "=")[1]
hash := matches[0]
m, _, err := client.DownloadManifest(hash)
if err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if len(m.Entries) != 1 {
t.Fatalf("expected one manifest entry, got %v", len(m.Entries))
}
e := m.Entries[0]
ct := "application/bzz-manifest+json"
if e.ContentType != ct {
t.Errorf("expected %q content type, got %q", ct, e.ContentType)
}
if e.Access == nil {
t.Fatal("manifest access is nil")
}
a := e.Access
if a.Type != "act" {
t.Fatalf(`got access type %q, expected "act"`, a.Type)
}
if len(a.Salt) < 32 {
t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt))
}
if a.Publisher != pkComp {
t.Fatal("publisher key did not match")
}
httpClient := &http.Client{}
// all nodes except the skipped node should be able to decrypt the content
for i, node := range cluster.Nodes {
log.Debug("trying to fetch from node", "node index", i)
url := node.URL + "/" + "bzz:/" + hash
response, err := httpClient.Get(url)
if err != nil {
t.Fatal(err)
}
log.Debug("got response from node", "response code", response.StatusCode)
if i == nodeToSkip {
log.Debug("reached node to skip", "status code", response.StatusCode)
if response.StatusCode != http.StatusUnauthorized {
t.Fatalf("should be a 401")
}
// try downloading using a password instead, using the unauthorized node
passwordUrl := strings.Replace(url, "http://", "http://:smth@", -1)
response, err = httpClient.Get(passwordUrl)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusOK {
t.Fatal("should be a 200")
}
// now try with the wrong password, expect 401
passwordUrl = strings.Replace(url, "http://", "http://:smthWrong@", -1)
response, err = httpClient.Get(passwordUrl)
if err != nil {
t.Fatal(err)
}
if response.StatusCode != http.StatusUnauthorized {
t.Fatal("should be a 401")
}
continue
}
if response.StatusCode != http.StatusOK {
t.Fatal("should be a 200")
}
d, err := ioutil.ReadAll(response.Body)
if err != nil {
t.Fatal(err)
}
if string(d) != data {
t.Errorf("expected decrypted data %q, got %q", data, string(d))
}
}
}
// TestKeypairSanity is a sanity test for the crypto scheme for ACT. it asserts the correct shared secret according to
// the specs at https://github.com/ethersphere/swarm-docs/blob/eb857afda906c6e7bb90d37f3f334ccce5eef230/act.md
func TestKeypairSanity(t *testing.T) {
salt := make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
t.Fatalf("reading from crypto/rand failed: %v", err.Error())
}
sharedSecret := "a85586744a1ddd56a7ed9f33fa24f40dd745b3a941be296a0d60e329dbdb896d"
for i, v := range []struct {
publisherPriv string
granteePub string
}{
{
publisherPriv: "ec5541555f3bc6376788425e9d1a62f55a82901683fd7062c5eddcc373a73459",
granteePub: "0226f213613e843a413ad35b40f193910d26eb35f00154afcde9ded57479a6224a",
},
{
publisherPriv: "70c7a73011aa56584a0009ab874794ee7e5652fd0c6911cd02f8b6267dd82d2d",
granteePub: "02e6f8d5e28faaa899744972bb847b6eb805a160494690c9ee7197ae9f619181db",
},
} {
b, _ := hex.DecodeString(v.granteePub)
granteePub, _ := crypto.DecompressPubkey(b)
publisherPrivate, _ := crypto.HexToECDSA(v.publisherPriv)
ssKey, err := api.NewSessionKeyPK(publisherPrivate, granteePub, salt)
if err != nil {
t.Fatal(err)
}
hasher := sha3.NewLegacyKeccak256()
hasher.Write(salt)
shared, err := hex.DecodeString(sharedSecret)
if err != nil {
t.Fatal(err)
}
hasher.Write(shared)
sum := hasher.Sum(nil)
if !bytes.Equal(ssKey, sum) {
t.Fatalf("%d: got a session key mismatch", i)
}
}
}

24
cmd/swarm/bootnodes.go
View file

@ -1,24 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
var SwarmBootnodes = []string{
// EF Swarm Bootnode - AWS - eu-central-1
"enode://4c113504601930bf2000c29bcd98d1716b6167749f58bad703bae338332fe93cc9d9204f08afb44100dc7bea479205f5d162df579f9a8f76f8b402d339709023@3.122.203.99:30301",
// EF Swarm Bootnode - AWS - us-west-2
"enode://89f2ede3371bff1ad9f2088f2012984e280287a4e2b68007c2a6ad994909c51886b4a8e9e2ecc97f9910aca538398e0a5804b0ee80a187fde1ba4f32626322ba@52.35.212.179:30301",
}

447
cmd/swarm/config.go
View file

@ -1,447 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"errors"
"fmt"
"io"
"os"
"reflect"
"strconv"
"strings"
"time"
"unicode"
cli "gopkg.in/urfave/cli.v1"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/node"
"github.com/naoina/toml"
bzzapi "github.com/ubiq/go-ubiq/swarm/api"
)
var (
//flag definition for the dumpconfig command
DumpConfigCommand = cli.Command{
Action: utils.MigrateFlags(dumpConfig),
Name: "dumpconfig",
Usage: "Show configuration values",
ArgsUsage: "",
Flags: app.Flags,
Category: "MISCELLANEOUS COMMANDS",
Description: `The dumpconfig command shows configuration values.`,
}
//flag definition for the config file command
SwarmTomlConfigPathFlag = cli.StringFlag{
Name: "config",
Usage: "TOML configuration file",
}
)
//constants for environment variables
const (
SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR"
SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT"
SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR"
SWARM_ENV_PORT = "SWARM_PORT"
SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID"
SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE"
SWARM_ENV_SWAP_API = "SWARM_SWAP_API"
SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE"
SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY"
SWARM_ENV_MAX_STREAM_PEER_SERVERS = "SWARM_ENV_MAX_STREAM_PEER_SERVERS"
SWARM_ENV_LIGHT_NODE_ENABLE = "SWARM_LIGHT_NODE_ENABLE"
SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK"
SWARM_ENV_ENS_API = "SWARM_ENS_API"
SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR"
SWARM_ENV_CORS = "SWARM_CORS"
SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES"
SWARM_ENV_PSS_ENABLE = "SWARM_PSS_ENABLE"
SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH"
SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY"
SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
SWARM_ENV_BOOTNODE_MODE = "SWARM_BOOTNODE_MODE"
SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD"
SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH"
SWARM_GLOBALSTORE_API = "SWARM_GLOBALSTORE_API"
GETH_ENV_DATADIR = "GETH_DATADIR"
)
// These settings ensure that TOML keys use the same names as Go struct fields.
var tomlSettings = toml.Config{
NormFieldName: func(rt reflect.Type, key string) string {
return key
},
FieldToKey: func(rt reflect.Type, field string) string {
return field
},
MissingField: func(rt reflect.Type, field string) error {
link := ""
if unicode.IsUpper(rune(rt.Name()[0])) && rt.PkgPath() != "main" {
link = fmt.Sprintf(", check github.com/ubiq/go-ubiq/swarm/api/config.go for available fields")
}
return fmt.Errorf("field '%s' is not defined in %s%s", field, rt.String(), link)
},
}
//before booting the swarm node, build the configuration
func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) {
//start by creating a default config
config = bzzapi.NewConfig()
//first load settings from config file (if provided)
config, err = configFileOverride(config, ctx)
if err != nil {
return nil, err
}
//override settings provided by environment variables
config = envVarsOverride(config)
//override settings provided by command line
config = cmdLineOverride(config, ctx)
//validate configuration parameters
err = validateConfig(config)
return
}
//finally, after the configuration build phase is finished, initialize
func initSwarmNode(config *bzzapi.Config, stack *node.Node, ctx *cli.Context) {
//at this point, all vars should be set in the Config
//get the account for the provided swarm account
prvkey := getAccount(config.BzzAccount, ctx, stack)
//set the resolved config path (gubiq --datadir)
config.Path = expandPath(stack.InstanceDir())
//finally, initialize the configuration
config.Init(prvkey)
//configuration phase completed here
log.Debug("Starting Swarm with the following parameters:")
//after having created the config, print it to screen
log.Debug(printConfig(config))
}
//configFileOverride overrides the current config with the config file, if a config file has been provided
func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config, error) {
var err error
//only do something if the -config flag has been set
if ctx.GlobalIsSet(SwarmTomlConfigPathFlag.Name) {
var filepath string
if filepath = ctx.GlobalString(SwarmTomlConfigPathFlag.Name); filepath == "" {
utils.Fatalf("Config file flag provided with invalid file path")
}
var f *os.File
f, err = os.Open(filepath)
if err != nil {
return nil, err
}
defer f.Close()
//decode the TOML file into a Config struct
//note that we are decoding into the existing defaultConfig;
//if an entry is not present in the file, the default entry is kept
err = tomlSettings.NewDecoder(f).Decode(&config)
// Add file name to errors that have a line number.
if _, ok := err.(*toml.LineError); ok {
err = errors.New(filepath + ", " + err.Error())
}
}
return config, err
}
// cmdLineOverride overrides the current config with whatever is provided through the command line
// most values are not allowed a zero value (empty string), if not otherwise noted
func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Config {
if keyid := ctx.GlobalString(SwarmAccountFlag.Name); keyid != "" {
currentConfig.BzzAccount = keyid
}
if chbookaddr := ctx.GlobalString(ChequebookAddrFlag.Name); chbookaddr != "" {
currentConfig.Contract = common.HexToAddress(chbookaddr)
}
if networkid := ctx.GlobalString(SwarmNetworkIdFlag.Name); networkid != "" {
id, err := strconv.ParseUint(networkid, 10, 64)
if err != nil {
utils.Fatalf("invalid cli flag %s: %v", SwarmNetworkIdFlag.Name, err)
}
if id != 0 {
currentConfig.NetworkID = id
}
}
if ctx.GlobalIsSet(utils.DataDirFlag.Name) {
if datadir := ctx.GlobalString(utils.DataDirFlag.Name); datadir != "" {
currentConfig.Path = expandPath(datadir)
}
}
bzzport := ctx.GlobalString(SwarmPortFlag.Name)
if len(bzzport) > 0 {
currentConfig.Port = bzzport
}
if bzzaddr := ctx.GlobalString(SwarmListenAddrFlag.Name); bzzaddr != "" {
currentConfig.ListenAddr = bzzaddr
}
if ctx.GlobalIsSet(SwarmSwapEnabledFlag.Name) {
currentConfig.SwapEnabled = true
}
if ctx.GlobalIsSet(SwarmSyncDisabledFlag.Name) {
currentConfig.SyncEnabled = false
}
if d := ctx.GlobalDuration(SwarmSyncUpdateDelay.Name); d > 0 {
currentConfig.SyncUpdateDelay = d
}
// any value including 0 is acceptable
currentConfig.MaxStreamPeerServers = ctx.GlobalInt(SwarmMaxStreamPeerServersFlag.Name)
if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) {
currentConfig.LightNodeEnabled = true
}
if ctx.GlobalIsSet(SwarmDeliverySkipCheckFlag.Name) {
currentConfig.DeliverySkipCheck = true
}
currentConfig.SwapAPI = ctx.GlobalString(SwarmSwapAPIFlag.Name)
if currentConfig.SwapEnabled && currentConfig.SwapAPI == "" {
utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
}
if ctx.GlobalIsSet(EnsAPIFlag.Name) {
ensAPIs := ctx.GlobalStringSlice(EnsAPIFlag.Name)
// preserve backward compatibility to disable ENS with --ens-api=""
if len(ensAPIs) == 1 && ensAPIs[0] == "" {
ensAPIs = nil
}
for i := range ensAPIs {
ensAPIs[i] = expandPath(ensAPIs[i])
}
currentConfig.EnsAPIs = ensAPIs
}
if cors := ctx.GlobalString(CorsStringFlag.Name); cors != "" {
currentConfig.Cors = cors
}
if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" {
currentConfig.LocalStoreParams.ChunkDbPath = storePath
}
if storeCapacity := ctx.GlobalUint64(SwarmStoreCapacity.Name); storeCapacity != 0 {
currentConfig.LocalStoreParams.DbCapacity = storeCapacity
}
if storeCacheCapacity := ctx.GlobalUint(SwarmStoreCacheCapacity.Name); storeCacheCapacity != 0 {
currentConfig.LocalStoreParams.CacheCapacity = storeCacheCapacity
}
if ctx.GlobalIsSet(SwarmBootnodeModeFlag.Name) {
currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name)
}
if ctx.GlobalIsSet(SwarmGlobalStoreAPIFlag.Name) {
currentConfig.GlobalStoreAPI = ctx.GlobalString(SwarmGlobalStoreAPIFlag.Name)
}
return currentConfig
}
// envVarsOverride overrides the current config with whatver is provided in environment variables
// most values are not allowed a zero value (empty string), if not otherwise noted
func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
if keyid := os.Getenv(SWARM_ENV_ACCOUNT); keyid != "" {
currentConfig.BzzAccount = keyid
}
if chbookaddr := os.Getenv(SWARM_ENV_CHEQUEBOOK_ADDR); chbookaddr != "" {
currentConfig.Contract = common.HexToAddress(chbookaddr)
}
if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" {
id, err := strconv.ParseUint(networkid, 10, 64)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_NETWORK_ID, err)
}
if id != 0 {
currentConfig.NetworkID = id
}
}
if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" {
currentConfig.Path = expandPath(datadir)
}
bzzport := os.Getenv(SWARM_ENV_PORT)
if len(bzzport) > 0 {
currentConfig.Port = bzzport
}
if bzzaddr := os.Getenv(SWARM_ENV_LISTEN_ADDR); bzzaddr != "" {
currentConfig.ListenAddr = bzzaddr
}
if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" {
swap, err := strconv.ParseBool(swapenable)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SWAP_ENABLE, err)
}
currentConfig.SwapEnabled = swap
}
if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" {
sync, err := strconv.ParseBool(syncdisable)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_DISABLE, err)
}
currentConfig.SyncEnabled = !sync
}
if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" {
skipCheck, err := strconv.ParseBool(v)
if err != nil {
currentConfig.DeliverySkipCheck = skipCheck
}
}
if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" {
d, err := time.ParseDuration(v)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_UPDATE_DELAY, err)
}
currentConfig.SyncUpdateDelay = d
}
if max := os.Getenv(SWARM_ENV_MAX_STREAM_PEER_SERVERS); max != "" {
m, err := strconv.Atoi(max)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_MAX_STREAM_PEER_SERVERS, err)
}
currentConfig.MaxStreamPeerServers = m
}
if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
lightnode, err := strconv.ParseBool(lne)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_LIGHT_NODE_ENABLE, err)
}
currentConfig.LightNodeEnabled = lightnode
}
if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {
currentConfig.SwapAPI = swapapi
}
if currentConfig.SwapEnabled && currentConfig.SwapAPI == "" {
utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
}
if ensapi := os.Getenv(SWARM_ENV_ENS_API); ensapi != "" {
currentConfig.EnsAPIs = strings.Split(ensapi, ",")
}
if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" {
currentConfig.EnsRoot = common.HexToAddress(ensaddr)
}
if cors := os.Getenv(SWARM_ENV_CORS); cors != "" {
currentConfig.Cors = cors
}
if bm := os.Getenv(SWARM_ENV_BOOTNODE_MODE); bm != "" {
bootnodeMode, err := strconv.ParseBool(bm)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_BOOTNODE_MODE, err)
}
currentConfig.BootnodeMode = bootnodeMode
}
if api := os.Getenv(SWARM_GLOBALSTORE_API); api != "" {
currentConfig.GlobalStoreAPI = api
}
return currentConfig
}
// dumpConfig is the dumpconfig command.
// writes a default config to STDOUT
func dumpConfig(ctx *cli.Context) error {
cfg, err := buildConfig(ctx)
if err != nil {
utils.Fatalf(fmt.Sprintf("Uh oh - dumpconfig triggered an error %v", err))
}
comment := ""
out, err := tomlSettings.Marshal(&cfg)
if err != nil {
return err
}
io.WriteString(os.Stdout, comment)
os.Stdout.Write(out)
return nil
}
//validate configuration parameters
func validateConfig(cfg *bzzapi.Config) (err error) {
for _, ensAPI := range cfg.EnsAPIs {
if ensAPI != "" {
if err := validateEnsAPIs(ensAPI); err != nil {
return fmt.Errorf("invalid format [tld:][contract-addr@]url for ENS API endpoint configuration %q: %v", ensAPI, err)
}
}
}
return nil
}
//validate EnsAPIs configuration parameter
func validateEnsAPIs(s string) (err error) {
// missing contract address
if strings.HasPrefix(s, "@") {
return errors.New("missing contract address")
}
// missing url
if strings.HasSuffix(s, "@") {
return errors.New("missing url")
}
// missing tld
if strings.HasPrefix(s, ":") {
return errors.New("missing tld")
}
// missing url
if strings.HasSuffix(s, ":") {
return errors.New("missing url")
}
return nil
}
//print a Config as string
func printConfig(config *bzzapi.Config) string {
out, err := tomlSettings.Marshal(&config)
if err != nil {
return fmt.Sprintf("Something is not right with the configuration: %v", err)
}
return string(out)
}

575
cmd/swarm/config_test.go
View file

@ -1,575 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"io"
"io/ioutil"
"net"
"os"
"os/exec"
"testing"
"time"
"github.com/docker/docker/pkg/reexec"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/rpc"
"github.com/ubiq/go-ubiq/swarm"
"github.com/ubiq/go-ubiq/swarm/api"
)
func TestConfigDump(t *testing.T) {
swarm := runSwarm(t, "dumpconfig")
defaultConf := api.NewConfig()
out, err := tomlSettings.Marshal(&defaultConf)
if err != nil {
t.Fatal(err)
}
swarm.Expect(string(out))
swarm.ExpectExit()
}
func TestConfigFailsSwapEnabledNoSwapApi(t *testing.T) {
flags := []string{
fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42",
fmt.Sprintf("--%s", SwarmPortFlag.Name), "54545",
fmt.Sprintf("--%s", SwarmSwapEnabledFlag.Name),
}
swarm := runSwarm(t, flags...)
swarm.Expect("Fatal: " + SWARM_ERR_SWAP_SET_NO_API + "\n")
swarm.ExpectExit()
}
func TestConfigFailsNoBzzAccount(t *testing.T) {
flags := []string{
fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42",
fmt.Sprintf("--%s", SwarmPortFlag.Name), "54545",
}
swarm := runSwarm(t, flags...)
swarm.Expect("Fatal: " + SWARM_ERR_NO_BZZACCOUNT + "\n")
swarm.ExpectExit()
}
func TestConfigCmdLineOverrides(t *testing.T) {
dir, err := ioutil.TempDir("", "bzztest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
conf, account := getTestAccount(t, dir)
node := &testNode{Dir: dir}
// assign ports
httpPort, err := assignTCPPort()
if err != nil {
t.Fatal(err)
}
flags := []string{
fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "42",
fmt.Sprintf("--%s", SwarmPortFlag.Name), httpPort,
fmt.Sprintf("--%s", SwarmSyncDisabledFlag.Name),
fmt.Sprintf("--%s", CorsStringFlag.Name), "*",
fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
fmt.Sprintf("--%s", SwarmDeliverySkipCheckFlag.Name),
fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
}
node.Cmd = runSwarm(t, flags...)
node.Cmd.InputLine(testPassphrase)
defer func() {
if t.Failed() {
node.Shutdown()
}
}()
// wait for the node to start
for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
node.Client, err = rpc.Dial(conf.IPCEndpoint())
if err == nil {
break
}
}
if node.Client == nil {
t.Fatal(err)
}
// load info
var info swarm.Info
if err := node.Client.Call(&info, "bzz_info"); err != nil {
t.Fatal(err)
}
if info.Port != httpPort {
t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port)
}
if info.NetworkID != 42 {
t.Fatalf("Expected network ID to be %d, got %d", 42, info.NetworkID)
}
if info.SyncEnabled {
t.Fatal("Expected Sync to be disabled, but is true")
}
if !info.DeliverySkipCheck {
t.Fatal("Expected DeliverySkipCheck to be enabled, but it is not")
}
if info.Cors != "*" {
t.Fatalf("Expected Cors flag to be set to %s, got %s", "*", info.Cors)
}
node.Shutdown()
}
func TestConfigFileOverrides(t *testing.T) {
// assign ports
httpPort, err := assignTCPPort()
if err != nil {
t.Fatal(err)
}
//create a config file
//first, create a default conf
defaultConf := api.NewConfig()
//change some values in order to test if they have been loaded
defaultConf.SyncEnabled = false
defaultConf.DeliverySkipCheck = true
defaultConf.NetworkID = 54
defaultConf.Port = httpPort
defaultConf.DbCapacity = 9000000
defaultConf.HiveParams.KeepAliveInterval = 6000000000
defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second
//defaultConf.SyncParams.KeyBufferSize = 512
//create a TOML string
out, err := tomlSettings.Marshal(&defaultConf)
if err != nil {
t.Fatalf("Error creating TOML file in TestFileOverride: %v", err)
}
//create file
f, err := ioutil.TempFile("", "testconfig.toml")
if err != nil {
t.Fatalf("Error writing TOML file in TestFileOverride: %v", err)
}
//write file
_, err = f.WriteString(string(out))
if err != nil {
t.Fatalf("Error writing TOML file in TestFileOverride: %v", err)
}
f.Sync()
dir, err := ioutil.TempDir("", "bzztest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
conf, account := getTestAccount(t, dir)
node := &testNode{Dir: dir}
flags := []string{
fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
}
node.Cmd = runSwarm(t, flags...)
node.Cmd.InputLine(testPassphrase)
defer func() {
if t.Failed() {
node.Shutdown()
}
}()
// wait for the node to start
for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
node.Client, err = rpc.Dial(conf.IPCEndpoint())
if err == nil {
break
}
}
if node.Client == nil {
t.Fatal(err)
}
// load info
var info swarm.Info
if err := node.Client.Call(&info, "bzz_info"); err != nil {
t.Fatal(err)
}
if info.Port != httpPort {
t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port)
}
if info.NetworkID != 54 {
t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkID)
}
if info.SyncEnabled {
t.Fatal("Expected Sync to be disabled, but is true")
}
if !info.DeliverySkipCheck {
t.Fatal("Expected DeliverySkipCheck to be enabled, but it is not")
}
if info.DbCapacity != 9000000 {
t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkID)
}
if info.HiveParams.KeepAliveInterval != 6000000000 {
t.Fatalf("Expected HiveParams KeepAliveInterval to be %d, got %d", uint64(6000000000), uint64(info.HiveParams.KeepAliveInterval))
}
if info.Swap.Params.Strategy.AutoCashInterval != 600*time.Second {
t.Fatalf("Expected SwapParams AutoCashInterval to be %ds, got %d", 600, info.Swap.Params.Strategy.AutoCashInterval)
}
// if info.SyncParams.KeyBufferSize != 512 {
// t.Fatalf("Expected info.SyncParams.KeyBufferSize to be %d, got %d", 512, info.SyncParams.KeyBufferSize)
// }
node.Shutdown()
}
func TestConfigEnvVars(t *testing.T) {
// assign ports
httpPort, err := assignTCPPort()
if err != nil {
t.Fatal(err)
}
envVars := os.Environ()
envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmPortFlag.EnvVar, httpPort))
envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmNetworkIdFlag.EnvVar, "999"))
envVars = append(envVars, fmt.Sprintf("%s=%s", CorsStringFlag.EnvVar, "*"))
envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmSyncDisabledFlag.EnvVar, "true"))
envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmDeliverySkipCheckFlag.EnvVar, "true"))
dir, err := ioutil.TempDir("", "bzztest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
conf, account := getTestAccount(t, dir)
node := &testNode{Dir: dir}
flags := []string{
fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
"--ens-api", "",
"--datadir", dir,
"--ipcpath", conf.IPCPath,
}
//node.Cmd = runSwarm(t,flags...)
//node.Cmd.cmd.Env = envVars
//the above assignment does not work, so we need a custom Cmd here in order to pass envVars:
cmd := &exec.Cmd{
Path: reexec.Self(),
Args: append([]string{"swarm-test"}, flags...),
Stderr: os.Stderr,
Stdout: os.Stdout,
}
cmd.Env = envVars
//stdout, err := cmd.StdoutPipe()
//if err != nil {
// t.Fatal(err)
//}
//stdout = bufio.NewReader(stdout)
var stdin io.WriteCloser
if stdin, err = cmd.StdinPipe(); err != nil {
t.Fatal(err)
}
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
//cmd.InputLine(testPassphrase)
io.WriteString(stdin, testPassphrase+"\n")
defer func() {
if t.Failed() {
node.Shutdown()
cmd.Process.Kill()
}
}()
// wait for the node to start
for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
node.Client, err = rpc.Dial(conf.IPCEndpoint())
if err == nil {
break
}
}
if node.Client == nil {
t.Fatal(err)
}
// load info
var info swarm.Info
if err := node.Client.Call(&info, "bzz_info"); err != nil {
t.Fatal(err)
}
if info.Port != httpPort {
t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port)
}
if info.NetworkID != 999 {
t.Fatalf("Expected network ID to be %d, got %d", 999, info.NetworkID)
}
if info.Cors != "*" {
t.Fatalf("Expected Cors flag to be set to %s, got %s", "*", info.Cors)
}
if info.SyncEnabled {
t.Fatal("Expected Sync to be disabled, but is true")
}
if !info.DeliverySkipCheck {
t.Fatal("Expected DeliverySkipCheck to be enabled, but it is not")
}
node.Shutdown()
cmd.Process.Kill()
}
func TestConfigCmdLineOverridesFile(t *testing.T) {
// assign ports
httpPort, err := assignTCPPort()
if err != nil {
t.Fatal(err)
}
//create a config file
//first, create a default conf
defaultConf := api.NewConfig()
//change some values in order to test if they have been loaded
defaultConf.SyncEnabled = true
defaultConf.NetworkID = 54
defaultConf.Port = "8588"
defaultConf.DbCapacity = 9000000
defaultConf.HiveParams.KeepAliveInterval = 6000000000
defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second
//defaultConf.SyncParams.KeyBufferSize = 512
//create a TOML file
out, err := tomlSettings.Marshal(&defaultConf)
if err != nil {
t.Fatalf("Error creating TOML file in TestFileOverride: %v", err)
}
//write file
fname := "testconfig.toml"
f, err := ioutil.TempFile("", fname)
if err != nil {
t.Fatalf("Error writing TOML file in TestFileOverride: %v", err)
}
defer os.Remove(fname)
//write file
_, err = f.WriteString(string(out))
if err != nil {
t.Fatalf("Error writing TOML file in TestFileOverride: %v", err)
}
f.Sync()
dir, err := ioutil.TempDir("", "bzztest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
conf, account := getTestAccount(t, dir)
node := &testNode{Dir: dir}
expectNetworkId := uint64(77)
flags := []string{
fmt.Sprintf("--%s", SwarmNetworkIdFlag.Name), "77",
fmt.Sprintf("--%s", SwarmPortFlag.Name), httpPort,
fmt.Sprintf("--%s", SwarmSyncDisabledFlag.Name),
fmt.Sprintf("--%s", SwarmTomlConfigPathFlag.Name), f.Name(),
fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(),
fmt.Sprintf("--%s", EnsAPIFlag.Name), "",
fmt.Sprintf("--%s", utils.DataDirFlag.Name), dir,
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), conf.IPCPath,
}
node.Cmd = runSwarm(t, flags...)
node.Cmd.InputLine(testPassphrase)
defer func() {
if t.Failed() {
node.Shutdown()
}
}()
// wait for the node to start
for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
node.Client, err = rpc.Dial(conf.IPCEndpoint())
if err == nil {
break
}
}
if node.Client == nil {
t.Fatal(err)
}
// load info
var info swarm.Info
if err := node.Client.Call(&info, "bzz_info"); err != nil {
t.Fatal(err)
}
if info.Port != httpPort {
t.Fatalf("Expected port to be %s, got %s", httpPort, info.Port)
}
if info.NetworkID != expectNetworkId {
t.Fatalf("Expected network ID to be %d, got %d", expectNetworkId, info.NetworkID)
}
if info.SyncEnabled {
t.Fatal("Expected Sync to be disabled, but is true")
}
if info.LocalStoreParams.DbCapacity != 9000000 {
t.Fatalf("Expected Capacity to be %d, got %d", 9000000, info.LocalStoreParams.DbCapacity)
}
if info.HiveParams.KeepAliveInterval != 6000000000 {
t.Fatalf("Expected HiveParams KeepAliveInterval to be %d, got %d", uint64(6000000000), uint64(info.HiveParams.KeepAliveInterval))
}
if info.Swap.Params.Strategy.AutoCashInterval != 600*time.Second {
t.Fatalf("Expected SwapParams AutoCashInterval to be %ds, got %d", 600, info.Swap.Params.Strategy.AutoCashInterval)
}
// if info.SyncParams.KeyBufferSize != 512 {
// t.Fatalf("Expected info.SyncParams.KeyBufferSize to be %d, got %d", 512, info.SyncParams.KeyBufferSize)
// }
node.Shutdown()
}
func TestConfigValidate(t *testing.T) {
for _, c := range []struct {
cfg *api.Config
err string
}{
{
cfg: &api.Config{EnsAPIs: []string{
"/data/testnet/gubiq.ipc",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"http://127.0.0.1:1234",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"ws://127.0.0.1:1234",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"test:/data/testnet/gubiq.ipc",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"test:ws://127.0.0.1:1234",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/gubiq.ipc",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"test:314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/gubiq.ipc",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"eth:314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"eth:314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:12344",
}},
},
{
cfg: &api.Config{EnsAPIs: []string{
"eth:",
}},
err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"eth:\": missing url",
},
{
cfg: &api.Config{EnsAPIs: []string{
"314159265dD8dbb310642f98f50C066173C1259b@",
}},
err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"314159265dD8dbb310642f98f50C066173C1259b@\": missing url",
},
{
cfg: &api.Config{EnsAPIs: []string{
":314159265dD8dbb310642f98f50C066173C1259",
}},
err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \":314159265dD8dbb310642f98f50C066173C1259\": missing tld",
},
{
cfg: &api.Config{EnsAPIs: []string{
"@/data/testnet/gubiq.ipc",
}},
err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"@/data/testnet/gubiq.ipc\": missing contract address",
},
} {
err := validateConfig(c.cfg)
if c.err != "" && err.Error() != c.err {
t.Errorf("expected error %q, got %q", c.err, err)
}
if c.err == "" && err != nil {
t.Errorf("unexpected error %q", err)
}
}
}
func assignTCPPort() (string, error) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return "", err
}
l.Close()
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
return "", err
}
return port, nil
}

147
cmd/swarm/db.go
View file

@ -1,147 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"io"
"os"
"path/filepath"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/swarm/storage"
"gopkg.in/urfave/cli.v1"
)
var dbCommand = cli.Command{
Name: "db",
CustomHelpTemplate: helpTemplate,
Usage: "manage the local chunk database",
ArgsUsage: "db COMMAND",
Description: "Manage the local chunk database",
Subcommands: []cli.Command{
{
Action: dbExport,
CustomHelpTemplate: helpTemplate,
Name: "export",
Usage: "export a local chunk database as a tar archive (use - to send to stdout)",
ArgsUsage: "<chunkdb> <file>",
Description: `
Export a local chunk database as a tar archive (use - to send to stdout).
swarm db export ~/.ubiq/swarm/bzz-KEY/chunks chunks.tar
The export may be quite large, consider piping the output through the Unix
pv(1) tool to get a progress bar:
swarm db export ~/.ubiq/swarm/bzz-KEY/chunks - | pv > chunks.tar
`,
},
{
Action: dbImport,
CustomHelpTemplate: helpTemplate,
Name: "import",
Usage: "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
ArgsUsage: "<chunkdb> <file>",
Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).
swarm db import ~/.ubiq/swarm/bzz-KEY/chunks chunks.tar
The import may be quite large, consider piping the input through the Unix
pv(1) tool to get a progress bar:
pv chunks.tar | swarm db import ~/.ubiq/swarm/bzz-KEY/chunks -`,
},
},
}
func dbExport(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 3 {
utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database), <file> (path to write the tar archive to, - for stdout) and the base key")
}
store, err := openLDBStore(args[0], common.Hex2Bytes(args[2]))
if err != nil {
utils.Fatalf("error opening local chunk database: %s", err)
}
defer store.Close()
var out io.Writer
if args[1] == "-" {
out = os.Stdout
} else {
f, err := os.Create(args[1])
if err != nil {
utils.Fatalf("error opening output file: %s", err)
}
defer f.Close()
out = f
}
count, err := store.Export(out)
if err != nil {
utils.Fatalf("error exporting local chunk database: %s", err)
}
log.Info(fmt.Sprintf("successfully exported %d chunks", count))
}
func dbImport(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 3 {
utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database), <file> (path to read the tar archive from, - for stdin) and the base key")
}
store, err := openLDBStore(args[0], common.Hex2Bytes(args[2]))
if err != nil {
utils.Fatalf("error opening local chunk database: %s", err)
}
defer store.Close()
var in io.Reader
if args[1] == "-" {
in = os.Stdin
} else {
f, err := os.Open(args[1])
if err != nil {
utils.Fatalf("error opening input file: %s", err)
}
defer f.Close()
in = f
}
count, err := store.Import(in)
if err != nil {
utils.Fatalf("error importing local chunk database: %s", err)
}
log.Info(fmt.Sprintf("successfully imported %d chunks", count))
}
func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) {
if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
return nil, fmt.Errorf("invalid chunkdb path: %s", err)
}
storeparams := storage.NewDefaultStoreParams()
ldbparams := storage.NewLDBStoreParams(storeparams, path)
ldbparams.BaseKey = basekey
return storage.NewLDBStore(ldbparams)
}

112
cmd/swarm/download.go
View file

@ -1,112 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/swarm/api"
swarm "github.com/ubiq/go-ubiq/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
var downloadCommand = cli.Command{
Action: download,
Name: "down",
Flags: []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag},
Usage: "downloads a swarm manifest or a file inside a manifest",
ArgsUsage: " <uri> [<dir>]",
Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`,
}
func download(ctx *cli.Context) {
log.Debug("downloading content using swarm down")
args := ctx.Args()
dest := "."
switch len(args) {
case 0:
utils.Fatalf("Usage: swarm down [options] <bzz locator> [<destination path>]")
case 1:
log.Trace(fmt.Sprintf("swarm down: no destination path - assuming working dir"))
default:
log.Trace(fmt.Sprintf("destination path arg: %s", args[1]))
if absDest, err := filepath.Abs(args[1]); err == nil {
dest = absDest
} else {
utils.Fatalf("could not get download path: %v", err)
}
}
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
isRecursive = ctx.Bool(SwarmRecursiveFlag.Name)
client = swarm.NewClient(bzzapi)
)
if fi, err := os.Stat(dest); err == nil {
if isRecursive && !fi.Mode().IsDir() {
utils.Fatalf("destination path is not a directory!")
}
} else {
if !os.IsNotExist(err) {
utils.Fatalf("could not stat path: %v", err)
}
}
uri, err := api.Parse(args[0])
if err != nil {
utils.Fatalf("could not parse uri argument: %v", err)
}
dl := func(credentials string) error {
// assume behaviour according to --recursive switch
if isRecursive {
if err := client.DownloadDirectory(uri.Addr, uri.Path, dest, credentials); err != nil {
if err == swarm.ErrUnauthorized {
return err
}
return fmt.Errorf("directory %s: %v", uri.Path, err)
}
} else {
// we are downloading a file
log.Debug("downloading file/path from a manifest", "uri.Addr", uri.Addr, "uri.Path", uri.Path)
err := client.DownloadFile(uri.Addr, uri.Path, dest, credentials)
if err != nil {
if err == swarm.ErrUnauthorized {
return err
}
return fmt.Errorf("file %s from address: %s: %v", uri.Path, uri.Addr, err)
}
}
return nil
}
if passwords := makePasswordList(ctx); passwords != nil {
password := getPassPhrase(fmt.Sprintf("Downloading %s is restricted", uri), 0, passwords)
err = dl(password)
} else {
err = dl("")
}
if err != nil {
utils.Fatalf("download: %v", err)
}
}

59
cmd/swarm/explore.go
View file

@ -1,59 +0,0 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Command bzzhash computes a swarm tree hash.
package main
import (
"context"
"fmt"
"os"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/swarm/storage"
"gopkg.in/urfave/cli.v1"
)
var hashesCommand = cli.Command{
Action: hashes,
CustomHelpTemplate: helpTemplate,
Name: "hashes",
Usage: "print all hashes of a file to STDOUT",
ArgsUsage: "<file>",
Description: "Prints all hashes of a file to STDOUT",
}
func hashes(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 1 {
utils.Fatalf("Usage: swarm hashes <file name>")
}
f, err := os.Open(args[0])
if err != nil {
utils.Fatalf("Error opening file " + args[1])
}
defer f.Close()
fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams())
refs, err := fileStore.GetAllReferences(context.TODO(), f, false)
if err != nil {
utils.Fatalf("%v\n", err)
} else {
for _, r := range refs {
fmt.Println(r.String())
}
}
}

119
cmd/swarm/export_test.go
View file

@ -1,119 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"crypto/md5"
"io"
"net/http"
"os"
"runtime"
"strings"
"testing"
"github.com/ubiq/go-ubiq/swarm"
"github.com/ubiq/go-ubiq/swarm/testutil"
)
// TestCLISwarmExportImport perform the following test:
// 1. runs swarm node
// 2. uploads a random file
// 3. runs an export of the local datastore
// 4. runs a second swarm node
// 5. imports the exported datastore
// 6. fetches the uploaded random file from the second node
func TestCLISwarmExportImport(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
cluster := newTestCluster(t, 1)
// generate random 1mb file
content := testutil.RandomBytes(1, 1000000)
fileName := testutil.TempFileWithContent(t, string(content))
defer os.Remove(fileName)
// upload the file with 'swarm up' and expect a hash
up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", fileName)
_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
up.ExpectExit()
hash := matches[0]
var info swarm.Info
if err := cluster.Nodes[0].Client.Call(&info, "bzz_info"); err != nil {
t.Fatal(err)
}
cluster.Stop()
defer cluster.Cleanup()
// generate an export.tar
exportCmd := runSwarm(t, "db", "export", info.Path+"/chunks", info.Path+"/export.tar", strings.TrimPrefix(info.BzzKey, "0x"))
exportCmd.ExpectExit()
// start second cluster
cluster2 := newTestCluster(t, 1)
var info2 swarm.Info
if err := cluster2.Nodes[0].Client.Call(&info2, "bzz_info"); err != nil {
t.Fatal(err)
}
// stop second cluster, so that we close LevelDB
cluster2.Stop()
defer cluster2.Cleanup()
// import the export.tar
importCmd := runSwarm(t, "db", "import", info2.Path+"/chunks", info.Path+"/export.tar", strings.TrimPrefix(info2.BzzKey, "0x"))
importCmd.ExpectExit()
// spin second cluster back up
cluster2.StartExistingNodes(t, 1, strings.TrimPrefix(info2.BzzAccount, "0x"))
// try to fetch imported file
res, err := http.Get(cluster2.Nodes[0].URL + "/bzz:/" + hash)
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 200 {
t.Fatalf("expected HTTP status %d, got %s", 200, res.Status)
}
// compare downloaded file with the generated random file
mustEqualFiles(t, bytes.NewReader(content), res.Body)
}
func mustEqualFiles(t *testing.T, up io.Reader, down io.Reader) {
h := md5.New()
upLen, err := io.Copy(h, up)
if err != nil {
t.Fatal(err)
}
upHash := h.Sum(nil)
h.Reset()
downLen, err := io.Copy(h, down)
if err != nil {
t.Fatal(err)
}
downHash := h.Sum(nil)
if !bytes.Equal(upHash, downHash) || upLen != downLen {
t.Fatalf("downloaded imported file md5=%x (length %v) is not the same as the generated one mp5=%x (length %v)", downHash, downLen, upHash, upLen)
}
}

238
cmd/swarm/feeds.go
View file

@ -1,238 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Command feed allows the user to create and update signed Swarm feeds
package main
import (
"fmt"
"strings"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/common/hexutil"
"github.com/ubiq/go-ubiq/crypto"
"github.com/ubiq/go-ubiq/cmd/utils"
swarm "github.com/ubiq/go-ubiq/swarm/api/client"
"github.com/ubiq/go-ubiq/swarm/storage/feed"
"gopkg.in/urfave/cli.v1"
)
var feedCommand = cli.Command{
CustomHelpTemplate: helpTemplate,
Name: "feed",
Usage: "(Advanced) Create and update Swarm Feeds",
ArgsUsage: "<create|update|info>",
Description: "Works with Swarm Feeds",
Subcommands: []cli.Command{
{
Action: feedCreateManifest,
CustomHelpTemplate: helpTemplate,
Name: "create",
Usage: "creates and publishes a new feed manifest",
Description: `creates and publishes a new feed manifest pointing to a specified user's updates about a particular topic.
The feed topic can be built in the following ways:
* use --topic to set the topic to an arbitrary binary hex string.
* use --name to set the topic to a human-readable name.
For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
* use both --topic and --name to create named subtopics.
For example, --topic could be set to an Ubiq contract address and --name could be set to "comments", meaning
this feed tracks a discussion about that contract.
The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
it will then default to your local account (--bzzaccount)`,
Flags: []cli.Flag{SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
},
{
Action: feedUpdate,
CustomHelpTemplate: helpTemplate,
Name: "update",
Usage: "updates the content of an existing Swarm Feed",
ArgsUsage: "<0x Hex data>",
Description: `publishes a new update on the specified topic
The feed topic can be built in the following ways:
* use --topic to set the topic to an arbitrary binary hex string.
* use --name to set the topic to a human-readable name.
For example --name could be set to "profile-picture", meaning this feed allows to get this user's current profile picture.
* use both --topic and --name to create named subtopics.
For example, --topic could be set to an Ubiq contract address and --name could be set to "comments", meaning
this feed tracks a discussion about that contract.
If you have a manifest, you can specify it with --manifest to refer to the feed,
instead of using --topic / --name
`,
Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag},
},
{
Action: feedInfo,
CustomHelpTemplate: helpTemplate,
Name: "info",
Usage: "obtains information about an existing Swarm feed",
Description: `obtains information about an existing Swarm feed
The topic can be specified directly with the --topic flag as an hex string
If no topic is specified, the default topic (zero) will be used
The --name flag can be used to specify subtopics with a specific name.
The --user flag allows to refer to a user other than yourself. If not specified,
it will then default to your local account (--bzzaccount)
If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
to refer to the feed`,
Flags: []cli.Flag{SwarmFeedManifestFlag, SwarmFeedNameFlag, SwarmFeedTopicFlag, SwarmFeedUserFlag},
},
},
}
func NewGenericSigner(ctx *cli.Context) feed.Signer {
return feed.NewGenericSigner(getPrivKey(ctx))
}
func getTopic(ctx *cli.Context) (topic feed.Topic) {
var name = ctx.String(SwarmFeedNameFlag.Name)
var relatedTopic = ctx.String(SwarmFeedTopicFlag.Name)
var relatedTopicBytes []byte
var err error
if relatedTopic != "" {
relatedTopicBytes, err = hexutil.Decode(relatedTopic)
if err != nil {
utils.Fatalf("Error parsing topic: %s", err)
}
}
topic, err = feed.NewTopic(name, relatedTopicBytes)
if err != nil {
utils.Fatalf("Error parsing topic: %s", err)
}
return topic
}
// swarm feed create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]]
// swarm feed update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false]
// swarm feed info <Manifest Address or ENS domain>
func feedCreateManifest(ctx *cli.Context) {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
)
newFeedUpdateRequest := feed.NewFirstRequest(getTopic(ctx))
newFeedUpdateRequest.Feed.User = feedGetUser(ctx)
manifestAddress, err := client.CreateFeedWithManifest(newFeedUpdateRequest)
if err != nil {
utils.Fatalf("Error creating feed manifest: %s", err.Error())
return
}
fmt.Println(manifestAddress) // output manifest address to the user in a single line (useful for other commands to pick up)
}
func feedUpdate(ctx *cli.Context) {
args := ctx.Args()
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name)
)
if len(args) < 1 {
fmt.Println("Incorrect number of arguments")
cli.ShowCommandHelpAndExit(ctx, "update", 1)
return
}
signer := NewGenericSigner(ctx)
data, err := hexutil.Decode(args[0])
if err != nil {
utils.Fatalf("Error parsing data: %s", err.Error())
return
}
var updateRequest *feed.Request
var query *feed.Query
if manifestAddressOrDomain == "" {
query = new(feed.Query)
query.User = signer.Address()
query.Topic = getTopic(ctx)
}
// Retrieve a feed update request
updateRequest, err = client.GetFeedRequest(query, manifestAddressOrDomain)
if err != nil {
utils.Fatalf("Error retrieving feed status: %s", err.Error())
}
// Check that the provided signer matches the request to sign
if updateRequest.User != signer.Address() {
utils.Fatalf("Signer address does not match the update request")
}
// set the new data
updateRequest.SetData(data)
// sign update
if err = updateRequest.Sign(signer); err != nil {
utils.Fatalf("Error signing feed update: %s", err.Error())
}
// post update
err = client.UpdateFeed(updateRequest)
if err != nil {
utils.Fatalf("Error updating feed: %s", err.Error())
return
}
}
func feedInfo(ctx *cli.Context) {
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client = swarm.NewClient(bzzapi)
manifestAddressOrDomain = ctx.String(SwarmFeedManifestFlag.Name)
)
var query *feed.Query
if manifestAddressOrDomain == "" {
query = new(feed.Query)
query.Topic = getTopic(ctx)
query.User = feedGetUser(ctx)
}
metadata, err := client.GetFeedRequest(query, manifestAddressOrDomain)
if err != nil {
utils.Fatalf("Error retrieving feed metadata: %s", err.Error())
return
}
encodedMetadata, err := metadata.MarshalJSON()
if err != nil {
utils.Fatalf("Error encoding metadata to JSON for display:%s", err)
}
fmt.Println(string(encodedMetadata))
}
func feedGetUser(ctx *cli.Context) common.Address {
var user = ctx.String(SwarmFeedUserFlag.Name)
if user != "" {
return common.HexToAddress(user)
}
pk := getPrivKey(ctx)
if pk == nil {
utils.Fatalf("Cannot read private key. Must specify --user or --bzzaccount")
}
return crypto.PubkeyToAddress(pk.PublicKey)
}

196
cmd/swarm/feeds_test.go
View file

@ -1,196 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"encoding/json"
"io/ioutil"
"os"
"testing"
"github.com/ubiq/go-ubiq/common/hexutil"
"github.com/ubiq/go-ubiq/crypto"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/swarm/api"
swarm "github.com/ubiq/go-ubiq/swarm/api/client"
swarmhttp "github.com/ubiq/go-ubiq/swarm/api/http"
"github.com/ubiq/go-ubiq/swarm/storage/feed"
"github.com/ubiq/go-ubiq/swarm/storage/feed/lookup"
"github.com/ubiq/go-ubiq/swarm/testutil"
)
func TestCLIFeedUpdate(t *testing.T) {
srv := swarmhttp.NewTestSwarmServer(t, func(api *api.API) swarmhttp.TestServer {
return swarmhttp.NewServer(api, "")
}, nil)
log.Info("starting a test swarm server")
defer srv.Close()
// create a private key file for signing
privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
privKey, _ := crypto.HexToECDSA(privkeyHex)
address := crypto.PubkeyToAddress(privKey.PublicKey)
pkFileName := testutil.TempFileWithContent(t, privkeyHex)
defer os.Remove(pkFileName)
// compose a topic. We'll be doing quotes about Miguel de Cervantes
var topic feed.Topic
subject := []byte("Miguel de Cervantes")
copy(topic[:], subject[:])
name := "quotes"
// prepare some data for the update
data := []byte("En boca cerrada no entran moscas")
hexData := hexutil.Encode(data)
flags := []string{
"--bzzapi", srv.URL,
"--bzzaccount", pkFileName,
"feed", "update",
"--topic", topic.Hex(),
"--name", name,
hexData}
// create an update and expect an exit without errors
log.Info("updating a feed with 'swarm feed update'")
cmd := runSwarm(t, flags...)
cmd.ExpectExit()
// now try to get the update using the client
client := swarm.NewClient(srv.URL)
// build the same topic as before, this time
// we use NewTopic to create a topic automatically.
topic, err := feed.NewTopic(name, subject)
if err != nil {
t.Fatal(err)
}
// Feed configures whose updates we will be looking up.
fd := feed.Feed{
Topic: topic,
User: address,
}
// Build a query to get the latest update
query := feed.NewQueryLatest(&fd, lookup.NoClue)
// retrieve content!
reader, err := client.QueryFeed(query, "")
if err != nil {
t.Fatal(err)
}
retrieved, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
// check we retrieved the sent information
if !bytes.Equal(data, retrieved) {
t.Fatalf("Received %s, expected %s", retrieved, data)
}
// Now retrieve info for the next update
flags = []string{
"--bzzapi", srv.URL,
"feed", "info",
"--topic", topic.Hex(),
"--user", address.Hex(),
}
log.Info("getting feed info with 'swarm feed info'")
cmd = runSwarm(t, flags...)
_, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout
cmd.ExpectExit()
// verify we can deserialize the result as a valid JSON
var request feed.Request
err = json.Unmarshal([]byte(matches[0]), &request)
if err != nil {
t.Fatal(err)
}
// make sure the retrieved feed is the same
if request.Feed != fd {
t.Fatalf("Expected feed to be: %s, got %s", fd, request.Feed)
}
// test publishing a manifest
flags = []string{
"--bzzapi", srv.URL,
"--bzzaccount", pkFileName,
"feed", "create",
"--topic", topic.Hex(),
}
log.Info("Publishing manifest with 'swarm feed create'")
cmd = runSwarm(t, flags...)
_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`)
cmd.ExpectExit()
manifestAddress := matches[0] // read the received feed manifest
// now attempt to lookup the latest update using a manifest instead
reader, err = client.QueryFeed(nil, manifestAddress)
if err != nil {
t.Fatal(err)
}
retrieved, err = ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, retrieved) {
t.Fatalf("Received %s, expected %s", retrieved, data)
}
// test publishing a manifest for a different user
flags = []string{
"--bzzapi", srv.URL,
"feed", "create",
"--topic", topic.Hex(),
"--user", "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", // different user
}
log.Info("Publishing manifest with 'swarm feed create' for a different user")
cmd = runSwarm(t, flags...)
_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`)
cmd.ExpectExit()
manifestAddress = matches[0] // read the received feed manifest
// now let's try to update that user's manifest which we don't have the private key for
flags = []string{
"--bzzapi", srv.URL,
"--bzzaccount", pkFileName,
"feed", "update",
"--manifest", manifestAddress,
hexData}
// create an update and expect an error given there is a user mismatch
log.Info("updating a feed with 'swarm feed update'")
cmd = runSwarm(t, flags...)
cmd.ExpectRegexp("Fatal:.*") // best way so far to detect a failure.
cmd.ExpectExit()
if cmd.ExitStatus() == 0 {
t.Fatal("Expected nonzero exit code when updating a manifest with the wrong user. Got 0.")
}
}

184
cmd/swarm/flags.go
View file

@ -1,184 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Command feed allows the user to create and update signed Swarm feeds
package main
import cli "gopkg.in/urfave/cli.v1"
var (
ChequebookAddrFlag = cli.StringFlag{
Name: "chequebook",
Usage: "chequebook contract address",
EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR,
}
SwarmAccountFlag = cli.StringFlag{
Name: "bzzaccount",
Usage: "Swarm account key file",
EnvVar: SWARM_ENV_ACCOUNT,
}
SwarmListenAddrFlag = cli.StringFlag{
Name: "httpaddr",
Usage: "Swarm HTTP API listening interface",
EnvVar: SWARM_ENV_LISTEN_ADDR,
}
SwarmPortFlag = cli.StringFlag{
Name: "bzzport",
Usage: "Swarm local http api port",
EnvVar: SWARM_ENV_PORT,
}
SwarmNetworkIdFlag = cli.IntFlag{
Name: "bzznetworkid",
Usage: "Network identifier (integer, default 3=swarm testnet)",
EnvVar: SWARM_ENV_NETWORK_ID,
}
SwarmSwapEnabledFlag = cli.BoolFlag{
Name: "swap",
Usage: "Swarm SWAP enabled (default false)",
EnvVar: SWARM_ENV_SWAP_ENABLE,
}
SwarmSwapAPIFlag = cli.StringFlag{
Name: "swap-api",
Usage: "URL of the Ethereum API provider to use to settle SWAP payments",
EnvVar: SWARM_ENV_SWAP_API,
}
SwarmSyncDisabledFlag = cli.BoolTFlag{
Name: "nosync",
Usage: "Disable swarm syncing",
EnvVar: SWARM_ENV_SYNC_DISABLE,
}
SwarmSyncUpdateDelay = cli.DurationFlag{
Name: "sync-update-delay",
Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)",
EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
}
SwarmMaxStreamPeerServersFlag = cli.IntFlag{
Name: "max-stream-peer-servers",
Usage: "Limit of Stream peer servers, 0 denotes unlimited",
EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS,
Value: 10000, // A very large default value is possible as stream servers have very small memory footprint
}
SwarmLightNodeEnabled = cli.BoolFlag{
Name: "lightnode",
Usage: "Enable Swarm LightNode (default false)",
EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
}
SwarmDeliverySkipCheckFlag = cli.BoolFlag{
Name: "delivery-skip-check",
Usage: "Skip chunk delivery check (default false)",
EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK,
}
EnsAPIFlag = cli.StringSliceFlag{
Name: "ens-api",
Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
EnvVar: SWARM_ENV_ENS_API,
}
SwarmApiFlag = cli.StringFlag{
Name: "bzzapi",
Usage: "Specifies the Swarm HTTP endpoint to connect to",
Value: "http://127.0.0.1:8500",
}
SwarmRecursiveFlag = cli.BoolFlag{
Name: "recursive",
Usage: "Upload directories recursively",
}
SwarmWantManifestFlag = cli.BoolTFlag{
Name: "manifest",
Usage: "Automatic manifest upload (default true)",
}
SwarmUploadDefaultPath = cli.StringFlag{
Name: "defaultpath",
Usage: "path to file served for empty url path (none)",
}
SwarmAccessGrantKeyFlag = cli.StringFlag{
Name: "grant-key",
Usage: "grants a given public key access to an ACT",
}
SwarmAccessGrantKeysFlag = cli.StringFlag{
Name: "grant-keys",
Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT",
}
SwarmUpFromStdinFlag = cli.BoolFlag{
Name: "stdin",
Usage: "reads data to be uploaded from stdin",
}
SwarmUploadMimeType = cli.StringFlag{
Name: "mime",
Usage: "Manually specify MIME type",
}
SwarmEncryptedFlag = cli.BoolFlag{
Name: "encrypt",
Usage: "use encrypted upload",
}
SwarmAccessPasswordFlag = cli.StringFlag{
Name: "password",
Usage: "Password",
EnvVar: SWARM_ACCESS_PASSWORD,
}
SwarmDryRunFlag = cli.BoolFlag{
Name: "dry-run",
Usage: "dry-run",
}
CorsStringFlag = cli.StringFlag{
Name: "corsdomain",
Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')",
EnvVar: SWARM_ENV_CORS,
}
SwarmStorePath = cli.StringFlag{
Name: "store.path",
Usage: "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)",
EnvVar: SWARM_ENV_STORE_PATH,
}
SwarmStoreCapacity = cli.Uint64Flag{
Name: "store.size",
Usage: "Number of chunks (5M is roughly 20-25GB) (default 5000000)",
EnvVar: SWARM_ENV_STORE_CAPACITY,
}
SwarmStoreCacheCapacity = cli.UintFlag{
Name: "store.cache.size",
Usage: "Number of recent chunks cached in memory (default 5000)",
EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
}
SwarmCompressedFlag = cli.BoolFlag{
Name: "compressed",
Usage: "Prints encryption keys in compressed form",
}
SwarmBootnodeModeFlag = cli.BoolFlag{
Name: "bootnode-mode",
Usage: "Run Swarm in Bootnode mode",
}
SwarmFeedNameFlag = cli.StringFlag{
Name: "name",
Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name",
}
SwarmFeedTopicFlag = cli.StringFlag{
Name: "topic",
Usage: "User-defined topic this feed is tracking, hex encoded. Limited to 64 hexadecimal characters",
}
SwarmFeedManifestFlag = cli.StringFlag{
Name: "manifest",
Usage: "Refers to the feed through a manifest",
}
SwarmFeedUserFlag = cli.StringFlag{
Name: "user",
Usage: "Indicates the user who updates the feed",
}
SwarmGlobalStoreAPIFlag = cli.StringFlag{
Name: "globalstore-api",
Usage: "URL of the Global Store API provider (only for testing)",
EnvVar: SWARM_GLOBALSTORE_API,
}
)

162
cmd/swarm/fs.go
View file

@ -1,162 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"fmt"
"path/filepath"
"strings"
"time"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/rpc"
"github.com/ubiq/go-ubiq/swarm/fuse"
"gopkg.in/urfave/cli.v1"
)
var fsCommand = cli.Command{
Name: "fs",
CustomHelpTemplate: helpTemplate,
Usage: "perform FUSE operations",
ArgsUsage: "fs COMMAND",
Description: "Performs FUSE operations by mounting/unmounting/listing mount points. This assumes you already have a Swarm node running locally. For all operation you must reference the correct path to bzzd.ipc in order to communicate with the node",
Subcommands: []cli.Command{
{
Action: mount,
CustomHelpTemplate: helpTemplate,
Name: "mount",
Usage: "mount a swarm hash to a mount point",
ArgsUsage: "swarm fs mount <manifest hash> <mount point>",
Description: "Mounts a Swarm manifest hash to a given mount point. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
},
{
Action: unmount,
CustomHelpTemplate: helpTemplate,
Name: "unmount",
Usage: "unmount a swarmfs mount",
ArgsUsage: "swarm fs unmount <mount point>",
Description: "Unmounts a swarmfs mount residing at <mount point>. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
},
{
Action: listMounts,
CustomHelpTemplate: helpTemplate,
Name: "list",
Usage: "list swarmfs mounts",
ArgsUsage: "swarm fs list",
Description: "Lists all mounted swarmfs volumes. This assumes you already have a Swarm node running locally. You must reference the correct path to your bzzd.ipc file",
},
},
}
func mount(cliContext *cli.Context) {
args := cliContext.Args()
if len(args) < 2 {
utils.Fatalf("Usage: swarm fs mount <manifestHash> <file name>")
}
client, err := dialRPC(cliContext)
if err != nil {
utils.Fatalf("had an error dailing to RPC endpoint: %v", err)
}
defer client.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
mf := &fuse.MountInfo{}
mountPoint, err := filepath.Abs(filepath.Clean(args[1]))
if err != nil {
utils.Fatalf("error expanding path for mount point: %v", err)
}
err = client.CallContext(ctx, mf, "swarmfs_mount", args[0], mountPoint)
if err != nil {
utils.Fatalf("had an error calling the RPC endpoint while mounting: %v", err)
}
}
func unmount(cliContext *cli.Context) {
args := cliContext.Args()
if len(args) < 1 {
utils.Fatalf("Usage: swarm fs unmount <mount path>")
}
client, err := dialRPC(cliContext)
if err != nil {
utils.Fatalf("had an error dailing to RPC endpoint: %v", err)
}
defer client.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
mf := fuse.MountInfo{}
err = client.CallContext(ctx, &mf, "swarmfs_unmount", args[0])
if err != nil {
utils.Fatalf("encountered an error calling the RPC endpoint while unmounting: %v", err)
}
fmt.Printf("%s\n", mf.LatestManifest) //print the latest manifest hash for user reference
}
func listMounts(cliContext *cli.Context) {
client, err := dialRPC(cliContext)
if err != nil {
utils.Fatalf("had an error dailing to RPC endpoint: %v", err)
}
defer client.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
mf := []fuse.MountInfo{}
err = client.CallContext(ctx, &mf, "swarmfs_listmounts")
if err != nil {
utils.Fatalf("encountered an error calling the RPC endpoint while listing mounts: %v", err)
}
if len(mf) == 0 {
fmt.Print("Could not found any swarmfs mounts. Please make sure you've specified the correct RPC endpoint\n")
} else {
fmt.Printf("Found %d swarmfs mount(s):\n", len(mf))
for i, mountInfo := range mf {
fmt.Printf("%d:\n", i)
fmt.Printf("\tMount point: %s\n", mountInfo.MountPoint)
fmt.Printf("\tLatest Manifest: %s\n", mountInfo.LatestManifest)
fmt.Printf("\tStart Manifest: %s\n", mountInfo.StartManifest)
}
}
}
func dialRPC(ctx *cli.Context) (*rpc.Client, error) {
endpoint := getIPCEndpoint(ctx)
log.Info("IPC endpoint", "path", endpoint)
return rpc.Dial(endpoint)
}
func getIPCEndpoint(ctx *cli.Context) string {
cfg := defaultNodeConfig
utils.SetNodeConfig(ctx, &cfg)
endpoint := cfg.IPCEndpoint()
if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") {
// Backwards compatibility with gubiq < 1.5 which required
// these prefixes.
endpoint = endpoint[4:]
}
return endpoint
}

260
cmd/swarm/fs_test.go
View file

@ -1,260 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// +build linux freebsd
package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/log"
)
type testFile struct {
filePath string
content string
}
// TestCLISwarmFsDefaultIPCPath tests if the most basic fs command, i.e., list
// can find and correctly connect to a running Swarm node on the default
// IPCPath.
func TestCLISwarmFsDefaultIPCPath(t *testing.T) {
cluster := newTestCluster(t, 1)
defer cluster.Shutdown()
handlingNode := cluster.Nodes[0]
list := runSwarm(t, []string{
"--datadir", handlingNode.Dir,
"fs",
"list",
}...)
list.WaitExit()
if list.Err != nil {
t.Fatal(list.Err)
}
}
// TestCLISwarmFs is a high-level test of swarmfs
//
// This test fails on travis for macOS as this executable exits with code 1
// and without any log messages in the log:
// /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse.
// This is the reason for this file not being built on darwin architecture.
func TestCLISwarmFs(t *testing.T) {
cluster := newTestCluster(t, 3)
defer cluster.Shutdown()
// create a tmp dir
mountPoint, err := ioutil.TempDir("", "swarm-test")
log.Debug("swarmfs cli test", "1st mount", mountPoint)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(mountPoint)
handlingNode := cluster.Nodes[0]
mhash := doUploadEmptyDir(t, handlingNode)
log.Debug("swarmfs cli test: mounting first run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
mount := runSwarm(t, []string{
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"mount",
mhash,
mountPoint,
}...)
mount.ExpectExit()
filesToAssert := []*testFile{}
dirPath, err := createDirInDir(mountPoint, "testSubDir")
if err != nil {
t.Fatal(err)
}
dirPath2, err := createDirInDir(dirPath, "AnotherTestSubDir")
if err != nil {
t.Fatal(err)
}
dummyContent := "somerandomtestcontentthatshouldbeasserted"
dirs := []string{
mountPoint,
dirPath,
dirPath2,
}
files := []string{"f1.tmp", "f2.tmp"}
for _, d := range dirs {
for _, entry := range files {
tFile, err := createTestFileInPath(d, entry, dummyContent)
if err != nil {
t.Fatal(err)
}
filesToAssert = append(filesToAssert, tFile)
}
}
if len(filesToAssert) != len(dirs)*len(files) {
t.Fatalf("should have %d files to assert now, got %d", len(dirs)*len(files), len(filesToAssert))
}
hashRegexp := `[a-f\d]{64}`
log.Debug("swarmfs cli test: unmounting first run...", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
unmount := runSwarm(t, []string{
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"unmount",
mountPoint,
}...)
_, matches := unmount.ExpectRegexp(hashRegexp)
unmount.ExpectExit()
hash := matches[0]
if hash == mhash {
t.Fatal("this should not be equal")
}
log.Debug("swarmfs cli test: asserting no files in mount point")
//check that there's nothing in the mount folder
filesInDir, err := ioutil.ReadDir(mountPoint)
if err != nil {
t.Fatalf("had an error reading the directory: %v", err)
}
if len(filesInDir) != 0 {
t.Fatal("there shouldn't be anything here")
}
secondMountPoint, err := ioutil.TempDir("", "swarm-test")
log.Debug("swarmfs cli test", "2nd mount point at", secondMountPoint)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(secondMountPoint)
log.Debug("swarmfs cli test: remounting at second mount point", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
//remount, check files
newMount := runSwarm(t, []string{
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"mount",
hash, // the latest hash
secondMountPoint,
}...)
newMount.ExpectExit()
time.Sleep(1 * time.Second)
filesInDir, err = ioutil.ReadDir(secondMountPoint)
if err != nil {
t.Fatal(err)
}
if len(filesInDir) == 0 {
t.Fatal("there should be something here")
}
log.Debug("swarmfs cli test: traversing file tree to see it matches previous mount")
for _, file := range filesToAssert {
file.filePath = strings.Replace(file.filePath, mountPoint, secondMountPoint, -1)
fileBytes, err := ioutil.ReadFile(file.filePath)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(fileBytes, bytes.NewBufferString(file.content).Bytes()) {
t.Fatal("this should be equal")
}
}
log.Debug("swarmfs cli test: unmounting second run", "ipc path", filepath.Join(handlingNode.Dir, handlingNode.IpcPath))
unmountSec := runSwarm(t, []string{
fmt.Sprintf("--%s", utils.IPCPathFlag.Name), filepath.Join(handlingNode.Dir, handlingNode.IpcPath),
"fs",
"unmount",
secondMountPoint,
}...)
_, matches = unmountSec.ExpectRegexp(hashRegexp)
unmountSec.ExpectExit()
if matches[0] != hash {
t.Fatal("these should be equal - no changes made")
}
}
func doUploadEmptyDir(t *testing.T, node *testNode) string {
// create a tmp dir
tmpDir, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
hashRegexp := `[a-f\d]{64}`
flags := []string{
"--bzzapi", node.URL,
"--recursive",
"up",
tmpDir}
log.Info("swarmfs cli test: uploading dir with 'swarm up'")
up := runSwarm(t, flags...)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
hash := matches[0]
log.Info("swarmfs cli test: dir uploaded", "hash", hash)
return hash
}
func createDirInDir(createInDir string, dirToCreate string) (string, error) {
fullpath := filepath.Join(createInDir, dirToCreate)
err := os.MkdirAll(fullpath, 0777)
if err != nil {
return "", err
}
return fullpath, nil
}
func createTestFileInPath(dir, filename, content string) (*testFile, error) {
tFile := &testFile{}
filePath := filepath.Join(dir, filename)
if file, err := os.Create(filePath); err == nil {
tFile.content = content
tFile.filePath = filePath
_, err = io.WriteString(file, content)
if err != nil {
return nil, err
}
file.Close()
}
return tFile, nil
}

100
cmd/swarm/global-store/global_store.go
View file

@ -1,100 +0,0 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"net"
"net/http"
"os"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/rpc"
"github.com/ubiq/go-ubiq/swarm/storage/mock"
"github.com/ubiq/go-ubiq/swarm/storage/mock/db"
"github.com/ubiq/go-ubiq/swarm/storage/mock/mem"
cli "gopkg.in/urfave/cli.v1"
)
// startHTTP starts a global store with HTTP RPC server.
// It is used for "http" cli command.
func startHTTP(ctx *cli.Context) (err error) {
server, cleanup, err := newServer(ctx)
if err != nil {
return err
}
defer cleanup()
listener, err := net.Listen("tcp", ctx.String("addr"))
if err != nil {
return err
}
log.Info("http", "address", listener.Addr().String())
return http.Serve(listener, server)
}
// startWS starts a global store with WebSocket RPC server.
// It is used for "websocket" cli command.
func startWS(ctx *cli.Context) (err error) {
server, cleanup, err := newServer(ctx)
if err != nil {
return err
}
defer cleanup()
listener, err := net.Listen("tcp", ctx.String("addr"))
if err != nil {
return err
}
origins := ctx.StringSlice("origins")
log.Info("websocket", "address", listener.Addr().String(), "origins", origins)
return http.Serve(listener, server.WebsocketHandler(origins))
}
// newServer creates a global store and returns its RPC server.
// Returned cleanup function should be called only if err is nil.
func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(false))))
cleanup = func() {}
var globalStore mock.GlobalStorer
dir := ctx.String("dir")
if dir != "" {
dbStore, err := db.NewGlobalStore(dir)
if err != nil {
return nil, nil, err
}
cleanup = func() {
dbStore.Close()
}
globalStore = dbStore
log.Info("database global store", "dir", dir)
} else {
globalStore = mem.NewGlobalStore()
log.Info("in-memory global store")
}
server = rpc.NewServer()
if err := server.RegisterName("mockStore", globalStore); err != nil {
cleanup()
return nil, nil, err
}
return server, cleanup, nil
}

191
cmd/swarm/global-store/global_store_test.go
View file

@ -1,191 +0,0 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"io/ioutil"
"net"
"net/http"
"os"
"testing"
"time"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/rpc"
mockRPC "github.com/ubiq/go-ubiq/swarm/storage/mock/rpc"
)
// TestHTTP_InMemory tests in-memory global store that exposes
// HTTP server.
func TestHTTP_InMemory(t *testing.T) {
testHTTP(t, true)
}
// TestHTTP_Database tests global store with persisted database
// that exposes HTTP server.
func TestHTTP_Database(t *testing.T) {
dir, err := ioutil.TempDir("", "swarm-global-store-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
// create a fresh global store
testHTTP(t, true, "--dir", dir)
// check if data saved by the previous global store instance
testHTTP(t, false, "--dir", dir)
}
// testWebsocket starts global store binary with HTTP server
// and validates that it can store and retrieve data.
// If put is false, no data will be stored, only retrieved,
// giving the possibility to check if data is present in the
// storage directory.
func testHTTP(t *testing.T, put bool, args ...string) {
addr := findFreeTCPAddress(t)
testCmd := runGlobalStore(t, append([]string{"http", "--addr", addr}, args...)...)
defer testCmd.Interrupt()
client, err := rpc.DialHTTP("http://" + addr)
if err != nil {
t.Fatal(err)
}
// wait until global store process is started as
// rpc.DialHTTP is actually not connecting
for i := 0; i < 1000; i++ {
_, err = http.DefaultClient.Get("http://" + addr)
if err == nil {
break
}
time.Sleep(10 * time.Millisecond)
}
if err != nil {
t.Fatal(err)
}
store := mockRPC.NewGlobalStore(client)
defer store.Close()
node := store.NewNodeStore(common.HexToAddress("123abc"))
wantKey := "key"
wantValue := "value"
if put {
err = node.Put([]byte(wantKey), []byte(wantValue))
if err != nil {
t.Fatal(err)
}
}
gotValue, err := node.Get([]byte(wantKey))
if err != nil {
t.Fatal(err)
}
if string(gotValue) != wantValue {
t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue)
}
}
// TestWebsocket_InMemory tests in-memory global store that exposes
// WebSocket server.
func TestWebsocket_InMemory(t *testing.T) {
testWebsocket(t, true)
}
// TestWebsocket_Database tests global store with persisted database
// that exposes HTTP server.
func TestWebsocket_Database(t *testing.T) {
dir, err := ioutil.TempDir("", "swarm-global-store-")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
// create a fresh global store
testWebsocket(t, true, "--dir", dir)
// check if data saved by the previous global store instance
testWebsocket(t, false, "--dir", dir)
}
// testWebsocket starts global store binary with WebSocket server
// and validates that it can store and retrieve data.
// If put is false, no data will be stored, only retrieved,
// giving the possibility to check if data is present in the
// storage directory.
func testWebsocket(t *testing.T, put bool, args ...string) {
addr := findFreeTCPAddress(t)
testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...)
defer testCmd.Interrupt()
var client *rpc.Client
var err error
// wait until global store process is started
for i := 0; i < 1000; i++ {
client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "")
if err == nil {
break
}
time.Sleep(10 * time.Millisecond)
}
if err != nil {
t.Fatal(err)
}
store := mockRPC.NewGlobalStore(client)
defer store.Close()
node := store.NewNodeStore(common.HexToAddress("123abc"))
wantKey := "key"
wantValue := "value"
if put {
err = node.Put([]byte(wantKey), []byte(wantValue))
if err != nil {
t.Fatal(err)
}
}
gotValue, err := node.Get([]byte(wantKey))
if err != nil {
t.Fatal(err)
}
if string(gotValue) != wantValue {
t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue)
}
}
// findFreeTCPAddress returns a local address (IP:Port) to which
// global store can listen on.
func findFreeTCPAddress(t *testing.T) (addr string) {
t.Helper()
listener, err := net.Listen("tcp", "")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
return listener.Addr().String()
}

104
cmd/swarm/global-store/main.go
View file

@ -1,104 +0,0 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"os"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/log"
cli "gopkg.in/urfave/cli.v1"
)
var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
func main() {
err := newApp().Run(os.Args)
if err != nil {
log.Error(err.Error())
os.Exit(1)
}
}
// newApp construct a new instance of Swarm Global Store.
// Method Run is called on it in the main function and in tests.
func newApp() (app *cli.App) {
app = utils.NewApp(gitCommit, "Swarm Global Store")
app.Name = "global-store"
// app flags (for all commands)
app.Flags = []cli.Flag{
cli.IntFlag{
Name: "verbosity",
Value: 3,
Usage: "verbosity level",
},
}
app.Commands = []cli.Command{
{
Name: "http",
Aliases: []string{"h"},
Usage: "start swarm global store with http server",
Action: startHTTP,
// Flags only for "start" command.
// Allow app flags to be specified after the
// command argument.
Flags: append(app.Flags,
cli.StringFlag{
Name: "dir",
Value: "",
Usage: "data directory",
},
cli.StringFlag{
Name: "addr",
Value: "0.0.0.0:3033",
Usage: "address to listen for http connection",
},
),
},
{
Name: "websocket",
Aliases: []string{"ws"},
Usage: "start swarm global store with websocket server",
Action: startWS,
// Flags only for "start" command.
// Allow app flags to be specified after the
// command argument.
Flags: append(app.Flags,
cli.StringFlag{
Name: "dir",
Value: "",
Usage: "data directory",
},
cli.StringFlag{
Name: "addr",
Value: "0.0.0.0:3033",
Usage: "address to listen for websocket connection",
},
cli.StringSliceFlag{
Name: "origins",
Value: &cli.StringSlice{"*"},
Usage: "websocket origins",
},
),
},
}
return app
}

49
cmd/swarm/global-store/run_test.go
View file

@ -1,49 +0,0 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"os"
"testing"
"github.com/docker/docker/pkg/reexec"
"github.com/ubiq/go-ubiq/internal/cmdtest"
)
func init() {
reexec.Register("swarm-global-store", func() {
if err := newApp().Run(os.Args); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
os.Exit(0)
})
}
func runGlobalStore(t *testing.T, args ...string) *cmdtest.TestCmd {
tt := cmdtest.NewTestCmd(t, nil)
tt.Run("swarm-global-store", args...)
return tt
}
func TestMain(m *testing.M) {
if reexec.Init() {
return
}
os.Exit(m.Run())
}

58
cmd/swarm/hash.go
View file

@ -1,58 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Command bzzhash computes a swarm tree hash.
package main
import (
"context"
"fmt"
"os"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/swarm/storage"
"gopkg.in/urfave/cli.v1"
)
var hashCommand = cli.Command{
Action: hash,
CustomHelpTemplate: helpTemplate,
Name: "hash",
Usage: "print the swarm hash of a file or directory",
ArgsUsage: "<file>",
Description: "Prints the swarm hash of file or directory",
}
func hash(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 1 {
utils.Fatalf("Usage: swarm hash <file name>")
}
f, err := os.Open(args[0])
if err != nil {
utils.Fatalf("Error opening file " + args[1])
}
defer f.Close()
stat, _ := f.Stat()
fileStore := storage.NewFileStore(&storage.FakeChunkStore{}, storage.NewFileStoreParams())
addr, _, err := fileStore.Store(context.TODO(), f, stat.Size(), false)
if err != nil {
utils.Fatalf("%v\n", err)
} else {
fmt.Printf("%v\n", addr)
}
}

70
cmd/swarm/list.go
View file

@ -1,70 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"os"
"strings"
"text/tabwriter"
"github.com/ubiq/go-ubiq/cmd/utils"
swarm "github.com/ubiq/go-ubiq/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
var listCommand = cli.Command{
Action: list,
CustomHelpTemplate: helpTemplate,
Name: "ls",
Usage: "list files and directories contained in a manifest",
ArgsUsage: "<manifest> [<prefix>]",
Description: "Lists files and directories contained in a manifest",
}
func list(ctx *cli.Context) {
args := ctx.Args()
if len(args) < 1 {
utils.Fatalf("Please supply a manifest reference as the first argument")
} else if len(args) > 2 {
utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
}
manifest := args[0]
var prefix string
if len(args) == 2 {
prefix = args[1]
}
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
list, err := client.List(manifest, prefix, "")
if err != nil {
utils.Fatalf("Failed to generate file and directory list: %s", err)
}
w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
defer w.Flush()
fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
for _, prefix := range list.CommonPrefixes {
fmt.Fprintf(w, "%s\t%s\t%s\n", "", "DIR", prefix)
}
for _, entry := range list.Entries {
fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
}
}

469
cmd/swarm/main.go
View file

@ -1,469 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"crypto/ecdsa"
"encoding/hex"
"fmt"
"io/ioutil"
"os"
"os/signal"
"runtime"
"sort"
"strconv"
"strings"
"syscall"
"github.com/ubiq/go-ubiq/accounts"
"github.com/ubiq/go-ubiq/accounts/keystore"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/console"
"github.com/ubiq/go-ubiq/crypto"
"github.com/ubiq/go-ubiq/internal/debug"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/node"
"github.com/ubiq/go-ubiq/p2p/enode"
"github.com/ubiq/go-ubiq/rpc"
"github.com/ubiq/go-ubiq/swarm"
bzzapi "github.com/ubiq/go-ubiq/swarm/api"
swarmmetrics "github.com/ubiq/go-ubiq/swarm/metrics"
"github.com/ubiq/go-ubiq/swarm/storage/mock"
mockrpc "github.com/ubiq/go-ubiq/swarm/storage/mock/rpc"
"github.com/ubiq/go-ubiq/swarm/tracing"
sv "github.com/ubiq/go-ubiq/swarm/version"
cli "gopkg.in/urfave/cli.v1"
)
const clientIdentifier = "swarm"
const helpTemplate = `NAME:
{{.HelpName}} - {{.Usage}}
USAGE:
{{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}}{{if .VisibleFlags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}}{{if .Category}}
CATEGORY:
{{.Category}}{{end}}{{if .Description}}
DESCRIPTION:
{{.Description}}{{end}}{{if .VisibleFlags}}
OPTIONS:
{{range .VisibleFlags}}{{.}}
{{end}}{{end}}
`
// Git SHA1 commit hash of the release (set via linker flags)
// this variable will be assigned if corresponding parameter is passed with install, but not with test
// e.g.: go install -ldflags "-X main.gitCommit=ed1312d01b19e04ef578946226e5d8069d5dfd5a" ./cmd/swarm
var gitCommit string
//declare a few constant error messages, useful for later error check comparisons in test
var (
SWARM_ERR_NO_BZZACCOUNT = "bzzaccount option is required but not set; check your config file, command line or environment variables"
SWARM_ERR_SWAP_SET_NO_API = "SWAP is enabled but --swap-api is not set"
)
// this help command gets added to any subcommand that does not define it explicitly
var defaultSubcommandHelp = cli.Command{
Action: func(ctx *cli.Context) { cli.ShowCommandHelpAndExit(ctx, "", 1) },
CustomHelpTemplate: helpTemplate,
Name: "help",
Usage: "shows this help",
Hidden: true,
}
var defaultNodeConfig = node.DefaultConfig
// This init function sets defaults so cmd/swarm can run alongside gubiq.
func init() {
sv.GitCommit = gitCommit
defaultNodeConfig.Name = clientIdentifier
defaultNodeConfig.Version = sv.VersionWithCommit(gitCommit)
defaultNodeConfig.P2P.ListenAddr = ":30399"
defaultNodeConfig.IPCPath = "bzzd.ipc"
// Set flag defaults for --help display.
utils.ListenPortFlag.Value = 30399
}
var app = utils.NewApp("", "Ubiq Swarm")
// This init function creates the cli.App.
func init() {
app.Action = bzzd
app.Version = sv.ArchiveVersion(gitCommit)
app.Copyright = "Copyright 2013-2016 The go-ubiq Authors"
app.Commands = []cli.Command{
{
Action: version,
CustomHelpTemplate: helpTemplate,
Name: "version",
Usage: "Print version numbers",
Description: "The output of this command is supposed to be machine-readable",
},
{
Action: keys,
CustomHelpTemplate: helpTemplate,
Name: "print-keys",
Flags: []cli.Flag{SwarmCompressedFlag},
Usage: "Print public key information",
Description: "The output of this command is supposed to be machine-readable",
},
// See upload.go
upCommand,
// See access.go
accessCommand,
// See feeds.go
feedCommand,
// See list.go
listCommand,
// See hash.go
hashCommand,
// See download.go
downloadCommand,
// See manifest.go
manifestCommand,
// See fs.go
fsCommand,
// See db.go
dbCommand,
// See config.go
DumpConfigCommand,
// hashesCommand
hashesCommand,
}
// append a hidden help subcommand to all commands that have subcommands
// if a help command was already defined above, that one will take precedence.
addDefaultHelpSubcommands(app.Commands)
sort.Sort(cli.CommandsByName(app.Commands))
app.Flags = []cli.Flag{
utils.IdentityFlag,
utils.DataDirFlag,
utils.BootnodesFlag,
utils.KeyStoreDirFlag,
utils.ListenPortFlag,
utils.DiscoveryV5Flag,
utils.NetrestrictFlag,
utils.NodeKeyFileFlag,
utils.NodeKeyHexFlag,
utils.MaxPeersFlag,
utils.NATFlag,
utils.IPCDisabledFlag,
utils.IPCPathFlag,
utils.PasswordFileFlag,
// bzzd-specific flags
CorsStringFlag,
EnsAPIFlag,
SwarmTomlConfigPathFlag,
SwarmSwapEnabledFlag,
SwarmSwapAPIFlag,
SwarmSyncDisabledFlag,
SwarmSyncUpdateDelay,
SwarmMaxStreamPeerServersFlag,
SwarmLightNodeEnabled,
SwarmDeliverySkipCheckFlag,
SwarmListenAddrFlag,
SwarmPortFlag,
SwarmAccountFlag,
SwarmNetworkIdFlag,
ChequebookAddrFlag,
// upload flags
SwarmApiFlag,
SwarmRecursiveFlag,
SwarmWantManifestFlag,
SwarmUploadDefaultPath,
SwarmUpFromStdinFlag,
SwarmUploadMimeType,
// bootnode mode
SwarmBootnodeModeFlag,
// storage flags
SwarmStorePath,
SwarmStoreCapacity,
SwarmStoreCacheCapacity,
SwarmGlobalStoreAPIFlag,
}
rpcFlags := []cli.Flag{
utils.WSEnabledFlag,
utils.WSListenAddrFlag,
utils.WSPortFlag,
utils.WSApiFlag,
utils.WSAllowedOriginsFlag,
}
app.Flags = append(app.Flags, rpcFlags...)
app.Flags = append(app.Flags, debug.Flags...)
app.Flags = append(app.Flags, swarmmetrics.Flags...)
app.Flags = append(app.Flags, tracing.Flags...)
app.Before = func(ctx *cli.Context) error {
runtime.GOMAXPROCS(runtime.NumCPU())
if err := debug.Setup(ctx, ""); err != nil {
return err
}
swarmmetrics.Setup(ctx)
tracing.Setup(ctx)
return nil
}
app.After = func(ctx *cli.Context) error {
debug.Exit()
return nil
}
}
func main() {
if err := app.Run(os.Args); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func keys(ctx *cli.Context) error {
privateKey := getPrivKey(ctx)
pubkey := crypto.FromECDSAPub(&privateKey.PublicKey)
pubkeyhex := hex.EncodeToString(pubkey)
pubCompressed := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey))
bzzkey := crypto.Keccak256Hash(pubkey).Hex()
if !ctx.Bool(SwarmCompressedFlag.Name) {
fmt.Println(fmt.Sprintf("bzzkey=%s", bzzkey[2:]))
fmt.Println(fmt.Sprintf("publicKey=%s", pubkeyhex))
}
fmt.Println(fmt.Sprintf("publicKeyCompressed=%s", pubCompressed))
return nil
}
func version(ctx *cli.Context) error {
fmt.Println(strings.Title(clientIdentifier))
fmt.Println("Version:", sv.VersionWithMeta)
if gitCommit != "" {
fmt.Println("Git Commit:", gitCommit)
}
fmt.Println("Go Version:", runtime.Version())
fmt.Println("OS:", runtime.GOOS)
return nil
}
func bzzd(ctx *cli.Context) error {
//build a valid bzzapi.Config from all available sources:
//default config, file config, command line and env vars
bzzconfig, err := buildConfig(ctx)
if err != nil {
utils.Fatalf("unable to configure swarm: %v", err)
}
cfg := defaultNodeConfig
//pss operates on ws
cfg.WSModules = append(cfg.WSModules, "pss")
//gubiq only supports --datadir via command line
//in order to be consistent within swarm, if we pass --datadir via environment variable
//or via config file, we get the same directory for gubiq and swarm
if _, err := os.Stat(bzzconfig.Path); err == nil {
cfg.DataDir = bzzconfig.Path
}
//optionally set the bootnodes before configuring the node
setSwarmBootstrapNodes(ctx, &cfg)
//setup the ethereum node
utils.SetNodeConfig(ctx, &cfg)
//always disable discovery from p2p package - swarm discovery is done with the `hive` protocol
cfg.P2P.NoDiscovery = true
stack, err := node.New(&cfg)
if err != nil {
utils.Fatalf("can't create node: %v", err)
}
//a few steps need to be done after the config phase is completed,
//due to overriding behavior
initSwarmNode(bzzconfig, stack, ctx)
//register BZZ as node.Service in the ethereum node
registerBzzService(bzzconfig, stack)
//start the node
utils.StartNode(stack)
go func() {
sigc := make(chan os.Signal, 1)
signal.Notify(sigc, syscall.SIGTERM)
defer signal.Stop(sigc)
<-sigc
log.Info("Got sigterm, shutting swarm down...")
stack.Stop()
}()
// add swarm bootnodes, because swarm doesn't use p2p package's discovery discv5
go func() {
s := stack.Server()
for _, n := range cfg.P2P.BootstrapNodes {
s.AddPeer(n)
}
}()
stack.Wait()
return nil
}
func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) {
//define the swarm service boot function
boot := func(_ *node.ServiceContext) (node.Service, error) {
var nodeStore *mock.NodeStore
if bzzconfig.GlobalStoreAPI != "" {
// connect to global store
client, err := rpc.Dial(bzzconfig.GlobalStoreAPI)
if err != nil {
return nil, fmt.Errorf("global store: %v", err)
}
globalStore := mockrpc.NewGlobalStore(client)
// create a node store for this swarm key on global store
nodeStore = globalStore.NewNodeStore(common.HexToAddress(bzzconfig.BzzKey))
}
return swarm.NewSwarm(bzzconfig, nodeStore)
}
//register within the ethereum node
if err := stack.Register(boot); err != nil {
utils.Fatalf("Failed to register the Swarm service: %v", err)
}
}
func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
//an account is mandatory
if bzzaccount == "" {
utils.Fatalf(SWARM_ERR_NO_BZZACCOUNT)
}
// Try to load the arg as a hex key file.
if key, err := crypto.LoadECDSA(bzzaccount); err == nil {
log.Info("Swarm account key loaded", "address", crypto.PubkeyToAddress(key.PublicKey))
return key
}
// Otherwise try getting it from the keystore.
am := stack.AccountManager()
ks := am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore)
return decryptStoreAccount(ks, bzzaccount, utils.MakePasswordList(ctx))
}
// getPrivKey returns the private key of the specified bzzaccount
// Used only by client commands, such as `feed`
func getPrivKey(ctx *cli.Context) *ecdsa.PrivateKey {
// booting up the swarm node just as we do in bzzd action
bzzconfig, err := buildConfig(ctx)
if err != nil {
utils.Fatalf("unable to configure swarm: %v", err)
}
cfg := defaultNodeConfig
if _, err := os.Stat(bzzconfig.Path); err == nil {
cfg.DataDir = bzzconfig.Path
}
utils.SetNodeConfig(ctx, &cfg)
stack, err := node.New(&cfg)
if err != nil {
utils.Fatalf("can't create node: %v", err)
}
return getAccount(bzzconfig.BzzAccount, ctx, stack)
}
func decryptStoreAccount(ks *keystore.KeyStore, account string, passwords []string) *ecdsa.PrivateKey {
var a accounts.Account
var err error
if common.IsHexAddress(account) {
a, err = ks.Find(accounts.Account{Address: common.HexToAddress(account)})
} else if ix, ixerr := strconv.Atoi(account); ixerr == nil && ix > 0 {
if accounts := ks.Accounts(); len(accounts) > ix {
a = accounts[ix]
} else {
err = fmt.Errorf("index %d higher than number of accounts %d", ix, len(accounts))
}
} else {
utils.Fatalf("Can't find swarm account key %s", account)
}
if err != nil {
utils.Fatalf("Can't find swarm account key: %v - Is the provided bzzaccount(%s) from the right datadir/Path?", err, account)
}
keyjson, err := ioutil.ReadFile(a.URL.Path)
if err != nil {
utils.Fatalf("Can't load swarm account key: %v", err)
}
for i := 0; i < 3; i++ {
password := getPassPhrase(fmt.Sprintf("Unlocking swarm account %s [%d/3]", a.Address.Hex(), i+1), i, passwords)
key, err := keystore.DecryptKey(keyjson, password)
if err == nil {
return key.PrivateKey
}
}
utils.Fatalf("Can't decrypt swarm account key")
return nil
}
// getPassPhrase retrieves the password associated with bzz account, either by fetching
// from a list of pre-loaded passwords, or by requesting it interactively from user.
func getPassPhrase(prompt string, i int, passwords []string) string {
// non-interactive
if len(passwords) > 0 {
if i < len(passwords) {
return passwords[i]
}
return passwords[len(passwords)-1]
}
// fallback to interactive mode
if prompt != "" {
fmt.Println(prompt)
}
password, err := console.Stdin.PromptPassword("Passphrase: ")
if err != nil {
utils.Fatalf("Failed to read passphrase: %v", err)
}
return password
}
// addDefaultHelpSubcommand scans through defined CLI commands and adds
// a basic help subcommand to each
// if a help command is already defined, it will take precedence over the default.
func addDefaultHelpSubcommands(commands []cli.Command) {
for i := range commands {
cmd := &commands[i]
if cmd.Subcommands != nil {
cmd.Subcommands = append(cmd.Subcommands, defaultSubcommandHelp)
addDefaultHelpSubcommands(cmd.Subcommands)
}
}
}
func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) {
if ctx.GlobalIsSet(utils.BootnodesFlag.Name) || ctx.GlobalIsSet(utils.BootnodesV4Flag.Name) {
return
}
cfg.P2P.BootstrapNodes = []*enode.Node{}
for _, url := range SwarmBootnodes {
node, err := enode.ParseV4(url)
if err != nil {
log.Error("Bootstrap URL invalid", "enode", url, "err", err)
}
cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node)
}
}

353
cmd/swarm/manifest.go
View file

@ -1,353 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Command MANIFEST update
package main
import (
"fmt"
"os"
"strings"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/swarm/api"
swarm "github.com/ubiq/go-ubiq/swarm/api/client"
"gopkg.in/urfave/cli.v1"
)
var manifestCommand = cli.Command{
Name: "manifest",
CustomHelpTemplate: helpTemplate,
Usage: "perform operations on swarm manifests",
ArgsUsage: "COMMAND",
Description: "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove",
Subcommands: []cli.Command{
{
Action: manifestAdd,
CustomHelpTemplate: helpTemplate,
Name: "add",
Usage: "add a new path to the manifest",
ArgsUsage: "<MANIFEST> <path> <hash>",
Description: "Adds a new path to the manifest",
},
{
Action: manifestUpdate,
CustomHelpTemplate: helpTemplate,
Name: "update",
Usage: "update the hash for an already existing path in the manifest",
ArgsUsage: "<MANIFEST> <path> <newhash>",
Description: "Update the hash for an already existing path in the manifest",
},
{
Action: manifestRemove,
CustomHelpTemplate: helpTemplate,
Name: "remove",
Usage: "removes a path from the manifest",
ArgsUsage: "<MANIFEST> <path>",
Description: "Removes a path from the manifest",
},
},
}
// manifestAdd adds a new entry to the manifest at the given path.
// New entry hash, the last argument, must be the hash of a manifest
// with only one entry, which meta-data will be added to the original manifest.
// On success, this function will print new (updated) manifest's hash.
func manifestAdd(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 3 {
utils.Fatalf("Need exactly three arguments <MHASH> <path> <HASH>")
}
var (
mhash = args[0]
path = args[1]
hash = args[2]
)
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
m, _, err := client.DownloadManifest(hash)
if err != nil {
utils.Fatalf("Error downloading manifest to add: %v", err)
}
l := len(m.Entries)
if l == 0 {
utils.Fatalf("No entries in manifest %s", hash)
} else if l > 1 {
utils.Fatalf("Too many entries in manifest %s", hash)
}
newManifest := addEntryToManifest(client, mhash, path, m.Entries[0])
fmt.Println(newManifest)
}
// manifestUpdate replaces an existing entry of the manifest at the given path.
// New entry hash, the last argument, must be the hash of a manifest
// with only one entry, which meta-data will be added to the original manifest.
// On success, this function will print hash of the updated manifest.
func manifestUpdate(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 3 {
utils.Fatalf("Need exactly three arguments <MHASH> <path> <HASH>")
}
var (
mhash = args[0]
path = args[1]
hash = args[2]
)
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
m, _, err := client.DownloadManifest(hash)
if err != nil {
utils.Fatalf("Error downloading manifest to update: %v", err)
}
l := len(m.Entries)
if l == 0 {
utils.Fatalf("No entries in manifest %s", hash)
} else if l > 1 {
utils.Fatalf("Too many entries in manifest %s", hash)
}
newManifest, _, defaultEntryUpdated := updateEntryInManifest(client, mhash, path, m.Entries[0], true)
if defaultEntryUpdated {
// Print informational message to stderr
// allowing the user to get the new manifest hash from stdout
// without the need to parse the complete output.
fmt.Fprintln(os.Stderr, "Manifest default entry is updated, too")
}
fmt.Println(newManifest)
}
// manifestRemove removes an existing entry of the manifest at the given path.
// On success, this function will print hash of the manifest which does not
// contain the path.
func manifestRemove(ctx *cli.Context) {
args := ctx.Args()
if len(args) != 2 {
utils.Fatalf("Need exactly two arguments <MHASH> <path>")
}
var (
mhash = args[0]
path = args[1]
)
bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
client := swarm.NewClient(bzzapi)
newManifest := removeEntryFromManifest(client, mhash, path)
fmt.Println(newManifest)
}
func addEntryToManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry) string {
var longestPathEntry = api.ManifestEntry{}
mroot, isEncrypted, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
// See if we path is in this Manifest or do we have to dig deeper
for _, e := range mroot.Entries {
if path == e.Path {
utils.Fatalf("Path %s already present, not adding anything", path)
} else {
if e.ContentType == api.ManifestType {
prfxlen := strings.HasPrefix(path, e.Path)
if prfxlen && len(path) > len(longestPathEntry.Path) {
longestPathEntry = e
}
}
}
}
if longestPathEntry.Path != "" {
// Load the child Manifest add the entry there
newPath := path[len(longestPathEntry.Path):]
newHash := addEntryToManifest(client, longestPathEntry.Hash, newPath, entry)
// Replace the hash for parent Manifests
newMRoot := &api.Manifest{}
for _, e := range mroot.Entries {
if longestPathEntry.Path == e.Path {
e.Hash = newHash
}
newMRoot.Entries = append(newMRoot.Entries, e)
}
mroot = newMRoot
} else {
// Add the entry in the leaf Manifest
entry.Path = path
mroot.Entries = append(mroot.Entries, entry)
}
newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
return newManifestHash
}
// updateEntryInManifest updates an existing entry o path with a new one in the manifest with provided mhash
// finding the path recursively through all nested manifests. Argument isRoot is used for default
// entry update detection. If the updated entry has the same hash as the default entry, then the
// default entry in root manifest will be updated too.
// Returned values are the new manifest hash, hash of the entry that was replaced by the new entry and
// a a bool that is true if default entry is updated.
func updateEntryInManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry, isRoot bool) (newManifestHash, oldHash string, defaultEntryUpdated bool) {
var (
newEntry = api.ManifestEntry{}
longestPathEntry = api.ManifestEntry{}
)
mroot, isEncrypted, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
// See if we path is in this Manifest or do we have to dig deeper
for _, e := range mroot.Entries {
if path == e.Path {
newEntry = e
// keep the reference of the hash of the entry that should be replaced
// for default entry detection
oldHash = e.Hash
} else {
if e.ContentType == api.ManifestType {
prfxlen := strings.HasPrefix(path, e.Path)
if prfxlen && len(path) > len(longestPathEntry.Path) {
longestPathEntry = e
}
}
}
}
if longestPathEntry.Path == "" && newEntry.Path == "" {
utils.Fatalf("Path %s not present in the Manifest, not setting anything", path)
}
if longestPathEntry.Path != "" {
// Load the child Manifest add the entry there
newPath := path[len(longestPathEntry.Path):]
var newHash string
newHash, oldHash, _ = updateEntryInManifest(client, longestPathEntry.Hash, newPath, entry, false)
// Replace the hash for parent Manifests
newMRoot := &api.Manifest{}
for _, e := range mroot.Entries {
if longestPathEntry.Path == e.Path {
e.Hash = newHash
}
newMRoot.Entries = append(newMRoot.Entries, e)
}
mroot = newMRoot
}
// update the manifest if the new entry is found and
// check if default entry should be updated
if newEntry.Path != "" || isRoot {
// Replace the hash for leaf Manifest
newMRoot := &api.Manifest{}
for _, e := range mroot.Entries {
if newEntry.Path == e.Path {
entry.Path = e.Path
newMRoot.Entries = append(newMRoot.Entries, entry)
} else if isRoot && e.Path == "" && e.Hash == oldHash {
entry.Path = e.Path
newMRoot.Entries = append(newMRoot.Entries, entry)
defaultEntryUpdated = true
} else {
newMRoot.Entries = append(newMRoot.Entries, e)
}
}
mroot = newMRoot
}
newManifestHash, err = client.UploadManifest(mroot, isEncrypted)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
return newManifestHash, oldHash, defaultEntryUpdated
}
func removeEntryFromManifest(client *swarm.Client, mhash, path string) string {
var (
entryToRemove = api.ManifestEntry{}
longestPathEntry = api.ManifestEntry{}
)
mroot, isEncrypted, err := client.DownloadManifest(mhash)
if err != nil {
utils.Fatalf("Manifest download failed: %v", err)
}
// See if we path is in this Manifest or do we have to dig deeper
for _, entry := range mroot.Entries {
if path == entry.Path {
entryToRemove = entry
} else {
if entry.ContentType == api.ManifestType {
prfxlen := strings.HasPrefix(path, entry.Path)
if prfxlen && len(path) > len(longestPathEntry.Path) {
longestPathEntry = entry
}
}
}
}
if longestPathEntry.Path == "" && entryToRemove.Path == "" {
utils.Fatalf("Path %s not present in the Manifest, not removing anything", path)
}
if longestPathEntry.Path != "" {
// Load the child Manifest remove the entry there
newPath := path[len(longestPathEntry.Path):]
newHash := removeEntryFromManifest(client, longestPathEntry.Hash, newPath)
// Replace the hash for parent Manifests
newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if longestPathEntry.Path == entry.Path {
entry.Hash = newHash
}
newMRoot.Entries = append(newMRoot.Entries, entry)
}
mroot = newMRoot
}
if entryToRemove.Path != "" {
// remove the entry in this Manifest
newMRoot := &api.Manifest{}
for _, entry := range mroot.Entries {
if entryToRemove.Path != entry.Path {
newMRoot.Entries = append(newMRoot.Entries, entry)
}
}
mroot = newMRoot
}
newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
if err != nil {
utils.Fatalf("Manifest upload failed: %v", err)
}
return newManifestHash
}

597
cmd/swarm/manifest_test.go
View file

@ -1,597 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"testing"
"github.com/ubiq/go-ubiq/swarm/api"
swarm "github.com/ubiq/go-ubiq/swarm/api/client"
swarmhttp "github.com/ubiq/go-ubiq/swarm/api/http"
)
// TestManifestChange tests manifest add, update and remove
// cli commands without encryption.
func TestManifestChange(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
testManifestChange(t, false)
}
// TestManifestChange tests manifest add, update and remove
// cli commands with encryption enabled.
func TestManifestChangeEncrypted(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
testManifestChange(t, true)
}
// testManifestChange performs cli commands:
// - manifest add
// - manifest update
// - manifest remove
// on a manifest, testing the functionality of this
// comands on paths that are in root manifest or a nested one.
// Argument encrypt controls whether to use encryption or not.
func testManifestChange(t *testing.T, encrypt bool) {
t.Parallel()
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
tmp, err := ioutil.TempDir("", "swarm-manifest-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
origDir := filepath.Join(tmp, "orig")
if err := os.Mkdir(origDir, 0777); err != nil {
t.Fatal(err)
}
indexDataFilename := filepath.Join(origDir, "index.html")
err = ioutil.WriteFile(indexDataFilename, []byte("<h1>Test</h1>"), 0666)
if err != nil {
t.Fatal(err)
}
// Files paths robots.txt and robots.html share the same prefix "robots."
// which will result a manifest with a nested manifest under path "robots.".
// This will allow testing manifest changes on both root and nested manifest.
err = ioutil.WriteFile(filepath.Join(origDir, "robots.txt"), []byte("Disallow: /"), 0666)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(origDir, "robots.html"), []byte("<strong>No Robots Allowed</strong>"), 0666)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(origDir, "mutants.txt"), []byte("Frank\nMarcus"), 0666)
if err != nil {
t.Fatal(err)
}
args := []string{
"--bzzapi",
srv.URL,
"--recursive",
"--defaultpath",
indexDataFilename,
"up",
origDir,
}
if encrypt {
args = append(args, "--encrypt")
}
origManifestHash := runSwarmExpectHash(t, args...)
checkHashLength(t, origManifestHash, encrypt)
client := swarm.NewClient(srv.URL)
// upload a new file and use its manifest to add it the original manifest.
t.Run("add", func(t *testing.T) {
humansData := []byte("Ann\nBob")
humansDataFilename := filepath.Join(tmp, "humans.txt")
err = ioutil.WriteFile(humansDataFilename, humansData, 0666)
if err != nil {
t.Fatal(err)
}
humansManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"up",
humansDataFilename,
)
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"manifest",
"add",
origManifestHash,
"humans.txt",
humansManifestHash,
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
for _, e := range newManifest.Entries {
if e.Path == "humans.txt" {
found = true
if e.Size != int64(len(humansData)) {
t.Errorf("expected humans.txt size %v, got %v", len(humansData), e.Size)
}
if e.ModTime.IsZero() {
t.Errorf("got zero mod time for humans.txt")
}
ct := "text/plain; charset=utf-8"
if e.ContentType != ct {
t.Errorf("expected content type %q, got %q", ct, e.ContentType)
}
break
}
}
if !found {
t.Fatal("no humans.txt in new manifest")
}
checkFile(t, client, newManifestHash, "humans.txt", humansData)
})
// upload a new file and use its manifest to add it the original manifest,
// but ensure that the file will be in the nested manifest of the original one.
t.Run("add nested", func(t *testing.T) {
robotsData := []byte(`{"disallow": "/"}`)
robotsDataFilename := filepath.Join(tmp, "robots.json")
err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666)
if err != nil {
t.Fatal(err)
}
robotsManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"up",
robotsDataFilename,
)
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"manifest",
"add",
origManifestHash,
"robots.json",
robotsManifestHash,
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
loop:
for _, e := range newManifest.Entries {
if e.Path == "robots." {
nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
for _, e := range nestedManifest.Entries {
if e.Path == "json" {
found = true
if e.Size != int64(len(robotsData)) {
t.Errorf("expected robots.json size %v, got %v", len(robotsData), e.Size)
}
if e.ModTime.IsZero() {
t.Errorf("got zero mod time for robots.json")
}
ct := "application/json"
if e.ContentType != ct {
t.Errorf("expected content type %q, got %q", ct, e.ContentType)
}
break loop
}
}
}
}
if !found {
t.Fatal("no robots.json in new manifest")
}
checkFile(t, client, newManifestHash, "robots.json", robotsData)
})
// upload a new file and use its manifest to change the file it the original manifest.
t.Run("update", func(t *testing.T) {
indexData := []byte("<h1>Ethereum Swarm</h1>")
indexDataFilename := filepath.Join(tmp, "index.html")
err = ioutil.WriteFile(indexDataFilename, indexData, 0666)
if err != nil {
t.Fatal(err)
}
indexManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"up",
indexDataFilename,
)
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"manifest",
"update",
origManifestHash,
"index.html",
indexManifestHash,
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
for _, e := range newManifest.Entries {
if e.Path == "index.html" {
found = true
if e.Size != int64(len(indexData)) {
t.Errorf("expected index.html size %v, got %v", len(indexData), e.Size)
}
if e.ModTime.IsZero() {
t.Errorf("got zero mod time for index.html")
}
ct := "text/html; charset=utf-8"
if e.ContentType != ct {
t.Errorf("expected content type %q, got %q", ct, e.ContentType)
}
break
}
}
if !found {
t.Fatal("no index.html in new manifest")
}
checkFile(t, client, newManifestHash, "index.html", indexData)
// check default entry change
checkFile(t, client, newManifestHash, "", indexData)
})
// upload a new file and use its manifest to change the file it the original manifest,
// but ensure that the file is in the nested manifest of the original one.
t.Run("update nested", func(t *testing.T) {
robotsData := []byte(`<string>Only humans allowed!!!</strong>`)
robotsDataFilename := filepath.Join(tmp, "robots.html")
err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666)
if err != nil {
t.Fatal(err)
}
humansManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"up",
robotsDataFilename,
)
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"manifest",
"update",
origManifestHash,
"robots.html",
humansManifestHash,
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
loop:
for _, e := range newManifest.Entries {
if e.Path == "robots." {
nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
for _, e := range nestedManifest.Entries {
if e.Path == "html" {
found = true
if e.Size != int64(len(robotsData)) {
t.Errorf("expected robots.html size %v, got %v", len(robotsData), e.Size)
}
if e.ModTime.IsZero() {
t.Errorf("got zero mod time for robots.html")
}
ct := "text/html; charset=utf-8"
if e.ContentType != ct {
t.Errorf("expected content type %q, got %q", ct, e.ContentType)
}
break loop
}
}
}
}
if !found {
t.Fatal("no robots.html in new manifest")
}
checkFile(t, client, newManifestHash, "robots.html", robotsData)
})
// remove a file from the manifest.
t.Run("remove", func(t *testing.T) {
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"manifest",
"remove",
origManifestHash,
"mutants.txt",
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
for _, e := range newManifest.Entries {
if e.Path == "mutants.txt" {
found = true
break
}
}
if found {
t.Fatal("mutants.txt is not removed")
}
})
// remove a file from the manifest, but ensure that the file is in
// the nested manifest of the original one.
t.Run("remove nested", func(t *testing.T) {
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"manifest",
"remove",
origManifestHash,
"robots.html",
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
loop:
for _, e := range newManifest.Entries {
if e.Path == "robots." {
nestedManifest := downloadManifest(t, client, e.Hash, encrypt)
for _, e := range nestedManifest.Entries {
if e.Path == "html" {
found = true
break loop
}
}
}
}
if found {
t.Fatal("robots.html in not removed")
}
})
}
// TestNestedDefaultEntryUpdate tests if the default entry is updated
// if the file in nested manifest used for it is also updated.
func TestNestedDefaultEntryUpdate(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
testNestedDefaultEntryUpdate(t, false)
}
// TestNestedDefaultEntryUpdateEncrypted tests if the default entry
// of encrypted upload is updated if the file in nested manifest
// used for it is also updated.
func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
testNestedDefaultEntryUpdate(t, true)
}
func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) {
t.Parallel()
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
tmp, err := ioutil.TempDir("", "swarm-manifest-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
origDir := filepath.Join(tmp, "orig")
if err := os.Mkdir(origDir, 0777); err != nil {
t.Fatal(err)
}
indexData := []byte("<h1>Test</h1>")
indexDataFilename := filepath.Join(origDir, "index.html")
err = ioutil.WriteFile(indexDataFilename, indexData, 0666)
if err != nil {
t.Fatal(err)
}
// Add another file with common prefix as the default entry to test updates of
// default entry with nested manifests.
err = ioutil.WriteFile(filepath.Join(origDir, "index.txt"), []byte("Test"), 0666)
if err != nil {
t.Fatal(err)
}
args := []string{
"--bzzapi",
srv.URL,
"--recursive",
"--defaultpath",
indexDataFilename,
"up",
origDir,
}
if encrypt {
args = append(args, "--encrypt")
}
origManifestHash := runSwarmExpectHash(t, args...)
checkHashLength(t, origManifestHash, encrypt)
client := swarm.NewClient(srv.URL)
newIndexData := []byte("<h1>Ethereum Swarm</h1>")
newIndexDataFilename := filepath.Join(tmp, "index.html")
err = ioutil.WriteFile(newIndexDataFilename, newIndexData, 0666)
if err != nil {
t.Fatal(err)
}
newIndexManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"up",
newIndexDataFilename,
)
newManifestHash := runSwarmExpectHash(t,
"--bzzapi",
srv.URL,
"manifest",
"update",
origManifestHash,
"index.html",
newIndexManifestHash,
)
checkHashLength(t, newManifestHash, encrypt)
newManifest := downloadManifest(t, client, newManifestHash, encrypt)
var found bool
for _, e := range newManifest.Entries {
if e.Path == "index." {
found = true
newManifest = downloadManifest(t, client, e.Hash, encrypt)
break
}
}
if !found {
t.Fatal("no index. path in new manifest")
}
found = false
for _, e := range newManifest.Entries {
if e.Path == "html" {
found = true
if e.Size != int64(len(newIndexData)) {
t.Errorf("expected index.html size %v, got %v", len(newIndexData), e.Size)
}
if e.ModTime.IsZero() {
t.Errorf("got zero mod time for index.html")
}
ct := "text/html; charset=utf-8"
if e.ContentType != ct {
t.Errorf("expected content type %q, got %q", ct, e.ContentType)
}
break
}
}
if !found {
t.Fatal("no html in new manifest")
}
checkFile(t, client, newManifestHash, "index.html", newIndexData)
// check default entry change
checkFile(t, client, newManifestHash, "", newIndexData)
}
func runSwarmExpectHash(t *testing.T, args ...string) (hash string) {
t.Helper()
hashRegexp := `[a-f\d]{64,128}`
up := runSwarm(t, args...)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
if len(matches) < 1 {
t.Fatal("no matches found")
}
return matches[0]
}
func checkHashLength(t *testing.T, hash string, encrypted bool) {
t.Helper()
l := len(hash)
if encrypted && l != 128 {
t.Errorf("expected hash length 128, got %v", l)
}
if !encrypted && l != 64 {
t.Errorf("expected hash length 64, got %v", l)
}
}
func downloadManifest(t *testing.T, client *swarm.Client, hash string, encrypted bool) (manifest *api.Manifest) {
t.Helper()
m, isEncrypted, err := client.DownloadManifest(hash)
if err != nil {
t.Fatal(err)
}
if encrypted != isEncrypted {
t.Error("new manifest encryption flag is not correct")
}
return m
}
func checkFile(t *testing.T, client *swarm.Client, hash, path string, expected []byte) {
t.Helper()
f, err := client.Download(hash, path)
if err != nil {
t.Fatal(err)
}
got, err := ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, expected) {
t.Errorf("expected file content %q, got %q", expected, got)
}
}

124
cmd/swarm/mimegen/generator.go
View file

@ -1,124 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
// Standard "mime" package rely on system-settings, see mime.osInitMime
// Swarm will run on many OS/Platform/Docker and must behave similar
// This command generates code to add common mime types based on mime.types file
//
// mime.types file provided by mailcap, which follow https://www.iana.org/assignments/media-types/media-types.xhtml
//
// Get last version of mime.types file by:
// docker run --rm -v $(pwd):/tmp alpine:edge /bin/sh -c "apk add -U mailcap; mv /etc/mime.types /tmp"
import (
"bufio"
"bytes"
"flag"
"html/template"
"io/ioutil"
"strings"
"log"
)
var (
typesFlag = flag.String("types", "", "Input mime.types file")
packageFlag = flag.String("package", "", "Golang package in output file")
outFlag = flag.String("out", "", "Output file name for the generated mime types")
)
type mime struct {
Name string
Exts []string
}
type templateParams struct {
PackageName string
Mimes []mime
}
func main() {
// Parse and ensure all needed inputs are specified
flag.Parse()
if *typesFlag == "" {
log.Fatalf("--types is required")
}
if *packageFlag == "" {
log.Fatalf("--types is required")
}
if *outFlag == "" {
log.Fatalf("--out is required")
}
params := templateParams{
PackageName: *packageFlag,
}
types, err := ioutil.ReadFile(*typesFlag)
if err != nil {
log.Fatal(err)
}
scanner := bufio.NewScanner(bytes.NewReader(types))
for scanner.Scan() {
txt := scanner.Text()
if strings.HasPrefix(txt, "#") || len(txt) == 0 {
continue
}
parts := strings.Fields(txt)
if len(parts) == 1 {
continue
}
params.Mimes = append(params.Mimes, mime{parts[0], parts[1:]})
}
if err = scanner.Err(); err != nil {
log.Fatal(err)
}
result := bytes.NewBuffer([]byte{})
if err := template.Must(template.New("_").Parse(tpl)).Execute(result, params); err != nil {
log.Fatal(err)
}
if err := ioutil.WriteFile(*outFlag, result.Bytes(), 0600); err != nil {
log.Fatal(err)
}
}
var tpl = `// Code generated by github.com/ubiq/go-ubiq/cmd/swarm/mimegen. DO NOT EDIT.
package {{ .PackageName }}
import "mime"
func init() {
var mimeTypes = map[string]string{
{{- range .Mimes -}}
{{ $name := .Name -}}
{{- range .Exts }}
".{{ . }}": "{{ $name | html }}",
{{- end }}
{{- end }}
}
for ext, name := range mimeTypes {
if err := mime.AddExtensionType(ext, name); err != nil {
panic(err)
}
}
}
`

1828
cmd/swarm/mimegen/mime.types
View file

File diff suppressed because it is too large Load diff

480
cmd/swarm/run_test.go
View file

@ -1,480 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"crypto/ecdsa"
"flag"
"fmt"
"io/ioutil"
"net"
"os"
"path"
"path/filepath"
"runtime"
"sync"
"syscall"
"testing"
"time"
"github.com/docker/docker/pkg/reexec"
"github.com/ubiq/go-ubiq/accounts"
"github.com/ubiq/go-ubiq/accounts/keystore"
"github.com/ubiq/go-ubiq/internal/cmdtest"
"github.com/ubiq/go-ubiq/node"
"github.com/ubiq/go-ubiq/p2p"
"github.com/ubiq/go-ubiq/rpc"
"github.com/ubiq/go-ubiq/swarm"
"github.com/ubiq/go-ubiq/swarm/api"
swarmhttp "github.com/ubiq/go-ubiq/swarm/api/http"
)
var loglevel = flag.Int("loglevel", 3, "verbosity of logs")
func init() {
// Run the app if we've been exec'd as "swarm-test" in runSwarm.
reexec.Register("swarm-test", func() {
if err := app.Run(os.Args); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
os.Exit(0)
})
}
const clusterSize = 3
var clusteronce sync.Once
var cluster *testCluster
func initCluster(t *testing.T) {
clusteronce.Do(func() {
cluster = newTestCluster(t, clusterSize)
})
}
func serverFunc(api *api.API) swarmhttp.TestServer {
return swarmhttp.NewServer(api, "")
}
func TestMain(m *testing.M) {
// check if we have been reexec'd
if reexec.Init() {
return
}
os.Exit(m.Run())
}
func runSwarm(t *testing.T, args ...string) *cmdtest.TestCmd {
tt := cmdtest.NewTestCmd(t, nil)
// Boot "swarm". This actually runs the test binary but the TestMain
// function will prevent any tests from running.
tt.Run("swarm-test", args...)
return tt
}
type testCluster struct {
Nodes []*testNode
TmpDir string
}
// newTestCluster starts a test swarm cluster of the given size.
//
// A temporary directory is created and each node gets a data directory inside
// it.
//
// Each node listens on 127.0.0.1 with random ports for both the HTTP and p2p
// ports (assigned by first listening on 127.0.0.1:0 and then passing the ports
// as flags).
//
// When starting more than one node, they are connected together using the
// admin SetPeer RPC method.
func newTestCluster(t *testing.T, size int) *testCluster {
cluster := &testCluster{}
defer func() {
if t.Failed() {
cluster.Shutdown()
}
}()
tmpdir, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
cluster.TmpDir = tmpdir
// start the nodes
cluster.StartNewNodes(t, size)
if size == 1 {
return cluster
}
// connect the nodes together
for _, node := range cluster.Nodes {
if err := node.Client.Call(nil, "admin_addPeer", cluster.Nodes[0].Enode); err != nil {
t.Fatal(err)
}
}
// wait until all nodes have the correct number of peers
outer:
for _, node := range cluster.Nodes {
var peers []*p2p.PeerInfo
for start := time.Now(); time.Since(start) < time.Minute; time.Sleep(50 * time.Millisecond) {
if err := node.Client.Call(&peers, "admin_peers"); err != nil {
t.Fatal(err)
}
if len(peers) == len(cluster.Nodes)-1 {
continue outer
}
}
t.Fatalf("%s only has %d / %d peers", node.Name, len(peers), len(cluster.Nodes)-1)
}
return cluster
}
func (c *testCluster) Shutdown() {
for _, node := range c.Nodes {
node.Shutdown()
}
os.RemoveAll(c.TmpDir)
}
func (c *testCluster) Stop() {
for _, node := range c.Nodes {
node.Shutdown()
}
}
func (c *testCluster) StartNewNodes(t *testing.T, size int) {
c.Nodes = make([]*testNode, 0, size)
for i := 0; i < size; i++ {
dir := filepath.Join(c.TmpDir, fmt.Sprintf("swarm%02d", i))
if err := os.Mkdir(dir, 0700); err != nil {
t.Fatal(err)
}
node := newTestNode(t, dir)
node.Name = fmt.Sprintf("swarm%02d", i)
c.Nodes = append(c.Nodes, node)
}
}
func (c *testCluster) StartExistingNodes(t *testing.T, size int, bzzaccount string) {
c.Nodes = make([]*testNode, 0, size)
for i := 0; i < size; i++ {
dir := filepath.Join(c.TmpDir, fmt.Sprintf("swarm%02d", i))
node := existingTestNode(t, dir, bzzaccount)
node.Name = fmt.Sprintf("swarm%02d", i)
c.Nodes = append(c.Nodes, node)
}
}
func (c *testCluster) Cleanup() {
os.RemoveAll(c.TmpDir)
}
type testNode struct {
Name string
Addr string
URL string
Enode string
Dir string
IpcPath string
PrivateKey *ecdsa.PrivateKey
Client *rpc.Client
Cmd *cmdtest.TestCmd
}
const testPassphrase = "swarm-test-passphrase"
func getTestAccount(t *testing.T, dir string) (conf *node.Config, account accounts.Account) {
// create key
conf = &node.Config{
DataDir: dir,
IPCPath: "bzzd.ipc",
NoUSB: true,
}
n, err := node.New(conf)
if err != nil {
t.Fatal(err)
}
account, err = n.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore).NewAccount(testPassphrase)
if err != nil {
t.Fatal(err)
}
// use a unique IPCPath when running tests on Windows
if runtime.GOOS == "windows" {
conf.IPCPath = fmt.Sprintf("bzzd-%s.ipc", account.Address.String())
}
return conf, account
}
func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
conf, _ := getTestAccount(t, dir)
node := &testNode{Dir: dir}
// use a unique IPCPath when running tests on Windows
if runtime.GOOS == "windows" {
conf.IPCPath = fmt.Sprintf("bzzd-%s.ipc", bzzaccount)
}
// assign ports
ports, err := getAvailableTCPPorts(2)
if err != nil {
t.Fatal(err)
}
p2pPort := ports[0]
httpPort := ports[1]
// start the node
node.Cmd = runSwarm(t,
"--port", p2pPort,
"--nat", "extip:127.0.0.1",
"--datadir", dir,
"--ipcpath", conf.IPCPath,
"--ens-api", "",
"--bzzaccount", bzzaccount,
"--bzznetworkid", "321",
"--bzzport", httpPort,
"--verbosity", fmt.Sprint(*loglevel),
)
node.Cmd.InputLine(testPassphrase)
defer func() {
if t.Failed() {
node.Shutdown()
}
}()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// ensure that all ports have active listeners
// so that the next node will not get the same
// when calling getAvailableTCPPorts
err = waitTCPPorts(ctx, ports...)
if err != nil {
t.Fatal(err)
}
// wait for the node to start
for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
node.Client, err = rpc.Dial(conf.IPCEndpoint())
if err == nil {
break
}
}
if node.Client == nil {
t.Fatal(err)
}
// load info
var info swarm.Info
if err := node.Client.Call(&info, "bzz_info"); err != nil {
t.Fatal(err)
}
node.Addr = net.JoinHostPort("127.0.0.1", info.Port)
node.URL = "http://" + node.Addr
var nodeInfo p2p.NodeInfo
if err := node.Client.Call(&nodeInfo, "admin_nodeInfo"); err != nil {
t.Fatal(err)
}
node.Enode = nodeInfo.Enode
node.IpcPath = conf.IPCPath
return node
}
func newTestNode(t *testing.T, dir string) *testNode {
conf, account := getTestAccount(t, dir)
ks := keystore.NewKeyStore(path.Join(dir, "keystore"), 1<<18, 1)
pk := decryptStoreAccount(ks, account.Address.Hex(), []string{testPassphrase})
node := &testNode{Dir: dir, PrivateKey: pk}
// assign ports
ports, err := getAvailableTCPPorts(2)
if err != nil {
t.Fatal(err)
}
p2pPort := ports[0]
httpPort := ports[1]
// start the node
node.Cmd = runSwarm(t,
"--port", p2pPort,
"--nat", "extip:127.0.0.1",
"--datadir", dir,
"--ipcpath", conf.IPCPath,
"--ens-api", "",
"--bzzaccount", account.Address.String(),
"--bzznetworkid", "321",
"--bzzport", httpPort,
"--verbosity", fmt.Sprint(*loglevel),
)
node.Cmd.InputLine(testPassphrase)
defer func() {
if t.Failed() {
node.Shutdown()
}
}()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// ensure that all ports have active listeners
// so that the next node will not get the same
// when calling getAvailableTCPPorts
err = waitTCPPorts(ctx, ports...)
if err != nil {
t.Fatal(err)
}
// wait for the node to start
for start := time.Now(); time.Since(start) < 10*time.Second; time.Sleep(50 * time.Millisecond) {
node.Client, err = rpc.Dial(conf.IPCEndpoint())
if err == nil {
break
}
}
if node.Client == nil {
t.Fatal(err)
}
// load info
var info swarm.Info
if err := node.Client.Call(&info, "bzz_info"); err != nil {
t.Fatal(err)
}
node.Addr = net.JoinHostPort("127.0.0.1", info.Port)
node.URL = "http://" + node.Addr
var nodeInfo p2p.NodeInfo
if err := node.Client.Call(&nodeInfo, "admin_nodeInfo"); err != nil {
t.Fatal(err)
}
node.Enode = nodeInfo.Enode
node.IpcPath = conf.IPCPath
return node
}
func (n *testNode) Shutdown() {
if n.Cmd != nil {
n.Cmd.Kill()
}
}
// getAvailableTCPPorts returns a set of ports that
// nothing is listening on at the time.
//
// Function assignTCPPort cannot be called in sequence
// and guardantee that the same port will be returned in
// different calls as the listener is closed within the function,
// not after all listeners are started and selected unique
// available ports.
func getAvailableTCPPorts(count int) (ports []string, err error) {
for i := 0; i < count; i++ {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return nil, err
}
// defer close in the loop to be sure the same port will not
// be selected in the next iteration
defer l.Close()
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
return nil, err
}
ports = append(ports, port)
}
return ports, nil
}
// waitTCPPorts blocks until tcp connections can be
// established on all provided ports. It runs all
// ports dialers in parallel, and returns the first
// encountered error.
// See waitTCPPort also.
func waitTCPPorts(ctx context.Context, ports ...string) error {
var err error
// mu locks err variable that is assigned in
// other goroutines
var mu sync.Mutex
// cancel is canceling all goroutines
// when the firs error is returned
// to prevent unnecessary waiting
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var wg sync.WaitGroup
for _, port := range ports {
wg.Add(1)
go func(port string) {
defer wg.Done()
e := waitTCPPort(ctx, port)
mu.Lock()
defer mu.Unlock()
if e != nil && err == nil {
err = e
cancel()
}
}(port)
}
wg.Wait()
return err
}
// waitTCPPort blocks until tcp connection can be established
// ona provided port. It has a 3 minute timeout as maximum,
// to prevent long waiting, but it can be shortened with
// a provided context instance. Dialer has a 10 second timeout
// in every iteration, and connection refused error will be
// retried in 100 milliseconds periods.
func waitTCPPort(ctx context.Context, port string) error {
ctx, cancel := context.WithTimeout(ctx, 3*time.Minute)
defer cancel()
for {
c, err := (&net.Dialer{Timeout: 10 * time.Second}).DialContext(ctx, "tcp", "127.0.0.1:"+port)
if err != nil {
if operr, ok := err.(*net.OpError); ok {
if syserr, ok := operr.Err.(*os.SyscallError); ok && syserr.Err == syscall.ECONNREFUSED {
time.Sleep(100 * time.Millisecond)
continue
}
}
return err
}
return c.Close()
}
}

291
cmd/swarm/swarm-smoke/feed_upload_and_sync.go
View file

@ -1,291 +0,0 @@
package main
import (
"bytes"
"crypto/md5"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"strings"
"sync"
"time"
"github.com/ubiq/go-ubiq/common/hexutil"
"github.com/ubiq/go-ubiq/crypto"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/swarm/storage/feed"
"github.com/ubiq/go-ubiq/swarm/testutil"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
const (
feedRandomDataLength = 8
)
func feedUploadAndSyncCmd(ctx *cli.Context, tuid string) error {
errc := make(chan error)
go func() {
errc <- feedUploadAndSync(ctx, tuid)
}()
select {
case err := <-errc:
if err != nil {
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
}
return err
case <-time.After(time.Duration(timeout) * time.Second):
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
return fmt.Errorf("timeout after %v sec", timeout)
}
}
func feedUploadAndSync(c *cli.Context, tuid string) error {
log.Info("generating and uploading feeds to " + httpEndpoint(hosts[0]) + " and syncing")
// create a random private key to sign updates with and derive the address
pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
if err != nil {
return err
}
defer pkFile.Close()
defer os.Remove(pkFile.Name())
privkeyHex := "0000000000000000000000000000000000000000000000000000000000001976"
privKey, err := crypto.HexToECDSA(privkeyHex)
if err != nil {
return err
}
user := crypto.PubkeyToAddress(privKey.PublicKey)
userHex := hexutil.Encode(user.Bytes())
// save the private key to a file
_, err = io.WriteString(pkFile, privkeyHex)
if err != nil {
return err
}
// keep hex strings for topic and subtopic
var topicHex string
var subTopicHex string
// and create combination hex topics for bzz-feed retrieval
// xor'ed with topic (zero-value topic if no topic)
var subTopicOnlyHex string
var mergedSubTopicHex string
// generate random topic and subtopic and put a hex on them
topicBytes, err := generateRandomData(feed.TopicLength)
topicHex = hexutil.Encode(topicBytes)
subTopicBytes, err := generateRandomData(8)
subTopicHex = hexutil.Encode(subTopicBytes)
if err != nil {
return err
}
mergedSubTopic, err := feed.NewTopic(subTopicHex, topicBytes)
if err != nil {
return err
}
mergedSubTopicHex = hexutil.Encode(mergedSubTopic[:])
subTopicOnlyBytes, err := feed.NewTopic(subTopicHex, nil)
if err != nil {
return err
}
subTopicOnlyHex = hexutil.Encode(subTopicOnlyBytes[:])
// create feed manifest, topic only
var out bytes.Buffer
cmd := exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--user", userHex)
cmd.Stdout = &out
log.Debug("create feed manifest topic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
return err
}
manifestWithTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
if len(manifestWithTopic) != 64 {
return fmt.Errorf("unknown feed create manifest hash format (topic): (%d) %s", len(out.String()), manifestWithTopic)
}
log.Debug("create topic feed", "manifest", manifestWithTopic)
out.Reset()
// create feed manifest, subtopic only
cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--name", subTopicHex, "--user", userHex)
cmd.Stdout = &out
log.Debug("create feed manifest subtopic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
return err
}
manifestWithSubTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
if len(manifestWithSubTopic) != 64 {
return fmt.Errorf("unknown feed create manifest hash format (subtopic): (%d) %s", len(out.String()), manifestWithSubTopic)
}
log.Debug("create subtopic feed", "manifest", manifestWithTopic)
out.Reset()
// create feed manifest, merged topic
cmd = exec.Command("swarm", "--bzzapi", httpEndpoint(hosts[0]), "feed", "create", "--topic", topicHex, "--name", subTopicHex, "--user", userHex)
cmd.Stdout = &out
log.Debug("create feed manifest mergetopic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
log.Error(err.Error())
return err
}
manifestWithMergedTopic := strings.TrimRight(out.String(), string([]byte{0x0a}))
if len(manifestWithMergedTopic) != 64 {
return fmt.Errorf("unknown feed create manifest hash format (mergedtopic): (%d) %s", len(out.String()), manifestWithMergedTopic)
}
log.Debug("create mergedtopic feed", "manifest", manifestWithMergedTopic)
out.Reset()
// create test data
data, err := generateRandomData(feedRandomDataLength)
if err != nil {
return err
}
h := md5.New()
h.Write(data)
dataHash := h.Sum(nil)
dataHex := hexutil.Encode(data)
// update with topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, dataHex)
cmd.Stdout = &out
log.Debug("update feed manifest topic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update topic", "out", out)
out.Reset()
// update with subtopic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, dataHex)
cmd.Stdout = &out
log.Debug("update feed manifest subtopic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update subtopic", "out", out)
out.Reset()
// update with merged topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, dataHex)
cmd.Stdout = &out
log.Debug("update feed manifest merged topic cmd", "cmd", cmd)
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update mergedtopic", "out", out)
out.Reset()
time.Sleep(3 * time.Second)
// retrieve the data
wg := sync.WaitGroup{}
for _, host := range hosts {
// raw retrieve, topic only
for _, hex := range []string{topicHex, subTopicOnlyHex, mergedSubTopicHex} {
wg.Add(1)
ruid := uuid.New()[:8]
go func(hex string, endpoint string, ruid string) {
for {
err := fetchFeed(hex, userHex, httpEndpoint(host), dataHash, ruid)
if err != nil {
continue
}
wg.Done()
return
}
}(hex, httpEndpoint(host), ruid)
}
}
wg.Wait()
log.Info("all endpoints synced random data successfully")
// upload test file
log.Info("feed uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed)
randomBytes := testutil.RandomBytes(seed, filesize*1000)
hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
if err != nil {
return err
}
hashBytes, err := hexutil.Decode("0x" + hash)
if err != nil {
return err
}
multihashHex := hexutil.Encode(hashBytes)
fileHash := h.Sum(nil)
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fileHash))
// update file with topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, multihashHex)
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update topic", "out", out)
out.Reset()
// update file with subtopic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--name", subTopicHex, multihashHex)
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update subtopic", "out", out)
out.Reset()
// update file with merged topic
cmd = exec.Command("swarm", "--bzzaccount", pkFile.Name(), "--bzzapi", httpEndpoint(hosts[0]), "feed", "update", "--topic", topicHex, "--name", subTopicHex, multihashHex)
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
return err
}
log.Debug("feed update mergedtopic", "out", out)
out.Reset()
time.Sleep(3 * time.Second)
for _, host := range hosts {
// manifest retrieve, topic only
for _, url := range []string{manifestWithTopic, manifestWithSubTopic, manifestWithMergedTopic} {
wg.Add(1)
ruid := uuid.New()[:8]
go func(url string, endpoint string, ruid string) {
for {
err := fetch(url, endpoint, fileHash, ruid, "")
if err != nil {
continue
}
wg.Done()
return
}
}(url, httpEndpoint(host), ruid)
}
}
wg.Wait()
log.Info("all endpoints synced random file successfully")
return nil
}

190
cmd/swarm/swarm-smoke/main.go
View file

@ -1,190 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"os"
"sort"
"github.com/ubiq/go-ubiq/cmd/utils"
gubiqmetrics "github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/metrics/influxdb"
swarmmetrics "github.com/ubiq/go-ubiq/swarm/metrics"
"github.com/ubiq/go-ubiq/swarm/tracing"
"github.com/ubiq/go-ubiq/log"
cli "gopkg.in/urfave/cli.v1"
)
var (
gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
)
var (
allhosts string
hosts []string
filesize int
syncDelay int
httpPort int
wsPort int
verbosity int
timeout int
single bool
trackTimeout int
)
func main() {
app := cli.NewApp()
app.Name = "smoke-test"
app.Usage = ""
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "hosts",
Value: "",
Usage: "comma-separated list of swarm hosts",
Destination: &allhosts,
},
cli.IntFlag{
Name: "http-port",
Value: 80,
Usage: "http port",
Destination: &httpPort,
},
cli.IntFlag{
Name: "ws-port",
Value: 8546,
Usage: "ws port",
Destination: &wsPort,
},
cli.IntFlag{
Name: "filesize",
Value: 1024,
Usage: "file size for generated random file in KB",
Destination: &filesize,
},
cli.IntFlag{
Name: "sync-delay",
Value: 5,
Usage: "duration of delay in seconds to wait for content to be synced",
Destination: &syncDelay,
},
cli.IntFlag{
Name: "verbosity",
Value: 1,
Usage: "verbosity",
Destination: &verbosity,
},
cli.IntFlag{
Name: "timeout",
Value: 120,
Usage: "timeout in seconds after which kill the process",
Destination: &timeout,
},
cli.BoolFlag{
Name: "single",
Usage: "whether to fetch content from a single node or from all nodes",
Destination: &single,
},
cli.IntFlag{
Name: "track-timeout",
Value: 5,
Usage: "timeout in seconds to wait for GetAllReferences to return",
Destination: &trackTimeout,
},
}
app.Flags = append(app.Flags, []cli.Flag{
utils.MetricsEnabledFlag,
swarmmetrics.MetricsInfluxDBEndpointFlag,
swarmmetrics.MetricsInfluxDBDatabaseFlag,
swarmmetrics.MetricsInfluxDBUsernameFlag,
swarmmetrics.MetricsInfluxDBPasswordFlag,
swarmmetrics.MetricsInfluxDBTagsFlag,
}...)
app.Flags = append(app.Flags, tracing.Flags...)
app.Commands = []cli.Command{
{
Name: "upload_and_sync",
Aliases: []string{"c"},
Usage: "upload and sync",
Action: wrapCliCommand("upload-and-sync", uploadAndSyncCmd),
},
{
Name: "feed_sync",
Aliases: []string{"f"},
Usage: "feed update generate, upload and sync",
Action: wrapCliCommand("feed-and-sync", feedUploadAndSyncCmd),
},
{
Name: "upload_speed",
Aliases: []string{"u"},
Usage: "measure upload speed",
Action: wrapCliCommand("upload-speed", uploadSpeedCmd),
},
{
Name: "sliding_window",
Aliases: []string{"s"},
Usage: "measure network aggregate capacity",
Action: wrapCliCommand("sliding-window", slidingWindowCmd),
},
}
sort.Sort(cli.FlagsByName(app.Flags))
sort.Sort(cli.CommandsByName(app.Commands))
app.Before = func(ctx *cli.Context) error {
tracing.Setup(ctx)
return nil
}
app.After = func(ctx *cli.Context) error {
return emitMetrics(ctx)
}
err := app.Run(os.Args)
if err != nil {
log.Error(err.Error())
os.Exit(1)
}
}
func emitMetrics(ctx *cli.Context) error {
if gubiqmetrics.Enabled {
var (
endpoint = ctx.GlobalString(swarmmetrics.MetricsInfluxDBEndpointFlag.Name)
database = ctx.GlobalString(swarmmetrics.MetricsInfluxDBDatabaseFlag.Name)
username = ctx.GlobalString(swarmmetrics.MetricsInfluxDBUsernameFlag.Name)
password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name)
tags = ctx.GlobalString(swarmmetrics.MetricsInfluxDBTagsFlag.Name)
)
tagsMap := utils.SplitTagsFlag(tags)
tagsMap["version"] = gitCommit
tagsMap["filesize"] = fmt.Sprintf("%v", filesize)
return influxdb.InfluxDBWithTagsOnce(gubiqmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", tagsMap)
}
return nil
}

131
cmd/swarm/swarm-smoke/sliding_window.go
View file

@ -1,131 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"fmt"
"math/rand"
"time"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/swarm/testutil"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
type uploadResult struct {
hash string
digest []byte
}
func slidingWindowCmd(ctx *cli.Context, tuid string) error {
errc := make(chan error)
go func() {
errc <- slidingWindow(ctx, tuid)
}()
select {
case err := <-errc:
if err != nil {
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
}
return err
case <-time.After(time.Duration(timeout) * time.Second):
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
return fmt.Errorf("timeout after %v sec", timeout)
}
}
func slidingWindow(ctx *cli.Context, tuid string) error {
hashes := []uploadResult{} //swarm hashes of the uploads
nodes := len(hosts)
const iterationTimeout = 30 * time.Second
log.Info("sliding window test started", "tuid", tuid, "nodes", nodes, "filesize(kb)", filesize, "timeout", timeout)
uploadedBytes := 0
networkDepth := 0
errored := false
outer:
for {
log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "seed", seed)
t1 := time.Now()
randomBytes := testutil.RandomBytes(seed, filesize*1000)
hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
if err != nil {
log.Error(err.Error())
return err
}
metrics.GetOrRegisterResettingTimer("sliding-window.upload-time", nil).UpdateSince(t1)
fhash, err := digest(bytes.NewReader(randomBytes))
if err != nil {
log.Error(err.Error())
return err
}
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash), "sleeping", syncDelay)
hashes = append(hashes, uploadResult{hash: hash, digest: fhash})
time.Sleep(time.Duration(syncDelay) * time.Second)
uploadedBytes += filesize * 1000
for i, v := range hashes {
timeout := time.After(time.Duration(timeout) * time.Second)
errored = false
inner:
for {
select {
case <-timeout:
errored = true
log.Error("error retrieving hash. timeout", "hash idx", i, "err", err)
metrics.GetOrRegisterCounter("sliding-window.single.error", nil).Inc(1)
break inner
default:
idx := 1 + rand.Intn(len(hosts)-1)
ruid := uuid.New()[:8]
start := time.Now()
err := fetch(v.hash, httpEndpoint(hosts[idx]), v.digest, ruid, "")
if err != nil {
continue inner
}
metrics.GetOrRegisterResettingTimer("sliding-window.single.fetch-time", nil).UpdateSince(start)
break inner
}
}
if errored {
break outer
}
networkDepth = i
metrics.GetOrRegisterGauge("sliding-window.network-depth", nil).Update(int64(networkDepth))
}
}
log.Info("sliding window test finished", "errored?", errored, "networkDepth", networkDepth, "networkDepth(kb)", networkDepth*filesize)
log.Info("stats", "uploadedFiles", len(hashes), "uploadedKb", uploadedBytes/1000, "filesizeKb", filesize)
return nil
}

194
cmd/swarm/swarm-smoke/upload_and_sync.go
View file

@ -1,194 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"math/rand"
"os"
"sync"
"time"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/rpc"
"github.com/ubiq/go-ubiq/swarm/api"
"github.com/ubiq/go-ubiq/swarm/storage"
"github.com/ubiq/go-ubiq/swarm/testutil"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
func uploadAndSyncCmd(ctx *cli.Context, tuid string) error {
randomBytes := testutil.RandomBytes(seed, filesize*1000)
errc := make(chan error)
go func() {
errc <- uplaodAndSync(ctx, randomBytes, tuid)
}()
select {
case err := <-errc:
if err != nil {
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
}
return err
case <-time.After(time.Duration(timeout) * time.Second):
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
e := fmt.Errorf("timeout after %v sec", timeout)
// trigger debug functionality on randomBytes
err := trackChunks(randomBytes[:])
if err != nil {
e = fmt.Errorf("%v; triggerChunkDebug failed: %v", e, err)
}
return e
}
}
func trackChunks(testData []byte) error {
log.Warn("Test timed out; running chunk debug sequence")
addrs, err := getAllRefs(testData)
if err != nil {
return err
}
log.Trace("All references retrieved")
// has-chunks
for _, host := range hosts {
httpHost := fmt.Sprintf("ws://%s:%d", host, 8546)
log.Trace("Calling `Has` on host", "httpHost", httpHost)
rpcClient, err := rpc.Dial(httpHost)
if err != nil {
log.Trace("Error dialing host", "err", err)
return err
}
log.Trace("rpc dial ok")
var hasInfo []api.HasInfo
err = rpcClient.Call(&hasInfo, "bzz_has", addrs)
if err != nil {
log.Trace("Error calling host", "err", err)
return err
}
log.Trace("rpc call ok")
count := 0
for _, info := range hasInfo {
if !info.Has {
count++
log.Error("Host does not have chunk", "host", httpHost, "chunk", info.Addr)
}
}
if count == 0 {
log.Info("Host reported to have all chunks", "host", httpHost)
}
}
return nil
}
func getAllRefs(testData []byte) (storage.AddressCollection, error) {
log.Trace("Getting all references for given root hash")
datadir, err := ioutil.TempDir("", "chunk-debug")
if err != nil {
return nil, fmt.Errorf("unable to create temp dir: %v", err)
}
defer os.RemoveAll(datadir)
fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32))
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(trackTimeout)*time.Second)
defer cancel()
reader := bytes.NewReader(testData)
return fileStore.GetAllReferences(ctx, reader, false)
}
func uplaodAndSync(c *cli.Context, randomBytes []byte, tuid string) error {
log.Info("uploading to "+httpEndpoint(hosts[0])+" and syncing", "tuid", tuid, "seed", seed)
t1 := time.Now()
hash, err := upload(randomBytes, httpEndpoint(hosts[0]))
if err != nil {
log.Error(err.Error())
return err
}
t2 := time.Since(t1)
metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).Update(t2)
fhash, err := digest(bytes.NewReader(randomBytes))
if err != nil {
log.Error(err.Error())
return err
}
log.Info("uploaded successfully", "tuid", tuid, "hash", hash, "took", t2, "digest", fmt.Sprintf("%x", fhash))
time.Sleep(time.Duration(syncDelay) * time.Second)
wg := sync.WaitGroup{}
if single {
randIndex := 1 + rand.Intn(len(hosts)-1)
ruid := uuid.New()[:8]
wg.Add(1)
go func(endpoint string, ruid string) {
for {
start := time.Now()
err := fetch(hash, endpoint, fhash, ruid, tuid)
if err != nil {
continue
}
ended := time.Since(start)
metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).Update(ended)
log.Info("fetch successful", "tuid", tuid, "ruid", ruid, "took", ended, "endpoint", endpoint)
wg.Done()
return
}
}(httpEndpoint(hosts[randIndex]), ruid)
} else {
for _, endpoint := range hosts[1:] {
ruid := uuid.New()[:8]
wg.Add(1)
go func(endpoint string, ruid string) {
for {
start := time.Now()
err := fetch(hash, endpoint, fhash, ruid, tuid)
if err != nil {
continue
}
ended := time.Since(start)
metrics.GetOrRegisterResettingTimer("upload-and-sync.each.fetch-time", nil).Update(ended)
log.Info("fetch successful", "tuid", tuid, "ruid", ruid, "took", ended, "endpoint", endpoint)
wg.Done()
return
}
}(httpEndpoint(endpoint), ruid)
}
}
wg.Wait()
log.Info("all hosts synced random file successfully")
return nil
}

73
cmd/swarm/swarm-smoke/upload_speed.go
View file

@ -1,73 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"fmt"
"time"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/swarm/testutil"
cli "gopkg.in/urfave/cli.v1"
)
func uploadSpeedCmd(ctx *cli.Context, tuid string) error {
log.Info("uploading to "+hosts[0], "tuid", tuid, "seed", seed)
randomBytes := testutil.RandomBytes(seed, filesize*1000)
errc := make(chan error)
go func() {
errc <- uploadSpeed(ctx, tuid, randomBytes)
}()
select {
case err := <-errc:
if err != nil {
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.fail", commandName), nil).Inc(1)
}
return err
case <-time.After(time.Duration(timeout) * time.Second):
metrics.GetOrRegisterCounter(fmt.Sprintf("%s.timeout", commandName), nil).Inc(1)
// trigger debug functionality on randomBytes
return fmt.Errorf("timeout after %v sec", timeout)
}
}
func uploadSpeed(c *cli.Context, tuid string, data []byte) error {
t1 := time.Now()
hash, err := upload(data, hosts[0])
if err != nil {
log.Error(err.Error())
return err
}
metrics.GetOrRegisterCounter("upload-speed.upload-time", nil).Inc(int64(time.Since(t1)))
fhash, err := digest(bytes.NewReader(data))
if err != nil {
log.Error(err.Error())
return err
}
log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash))
return nil
}

235
cmd/swarm/swarm-smoke/util.go
View file

@ -1,235 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"context"
"crypto/md5"
crand "crypto/rand"
"errors"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/http"
"net/http/httptrace"
"os"
"strings"
"time"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/swarm/api"
"github.com/ubiq/go-ubiq/swarm/api/client"
"github.com/ubiq/go-ubiq/swarm/spancontext"
opentracing "github.com/opentracing/opentracing-go"
"github.com/pborman/uuid"
cli "gopkg.in/urfave/cli.v1"
)
var (
commandName = ""
seed = int(time.Now().UTC().UnixNano())
)
func init() {
rand.Seed(int64(seed))
}
func httpEndpoint(host string) string {
return fmt.Sprintf("http://%s:%d", host, httpPort)
}
func wsEndpoint(host string) string {
return fmt.Sprintf("ws://%s:%d", host, wsPort)
}
func wrapCliCommand(name string, command func(*cli.Context, string) error) func(*cli.Context) error {
return func(ctx *cli.Context) error {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(false))))
// test uuid
tuid := uuid.New()[:8]
commandName = name
hosts = strings.Split(allhosts, ",")
defer func(now time.Time) {
totalTime := time.Since(now)
log.Info("total time", "tuid", tuid, "time", totalTime, "kb", filesize)
metrics.GetOrRegisterResettingTimer(name+".total-time", nil).Update(totalTime)
}(time.Now())
log.Info("smoke test starting", "tuid", tuid, "task", name, "timeout", timeout)
metrics.GetOrRegisterCounter(name, nil).Inc(1)
return command(ctx, tuid)
}
}
func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error {
ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch")
defer sp.Finish()
log.Trace("sleeping", "ruid", ruid)
time.Sleep(3 * time.Second)
log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user)
var tn time.Time
reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user
req, _ := http.NewRequest("GET", reqUri, nil)
opentracing.GlobalTracer().Inject(
sp.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header))
trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn)
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
transport := http.DefaultTransport
//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
tn = time.Now()
res, err := transport.RoundTrip(req)
if err != nil {
log.Error(err.Error(), "ruid", ruid)
return err
}
log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength)
if res.StatusCode != 200 {
return fmt.Errorf("expected status code %d, got %v (ruid %v)", 200, res.StatusCode, ruid)
}
defer res.Body.Close()
rdigest, err := digest(res.Body)
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err
}
if !bytes.Equal(rdigest, original) {
err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
log.Warn(err.Error(), "ruid", ruid)
return err
}
log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
return nil
}
// fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file
func fetch(hash string, endpoint string, original []byte, ruid string, tuid string) error {
ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch")
defer sp.Finish()
log.Info("http get request", "tuid", tuid, "ruid", ruid, "endpoint", endpoint, "hash", hash)
var tn time.Time
reqUri := endpoint + "/bzz:/" + hash + "/"
req, _ := http.NewRequest("GET", reqUri, nil)
opentracing.GlobalTracer().Inject(
sp.Context(),
opentracing.HTTPHeaders,
opentracing.HTTPHeadersCarrier(req.Header))
trace := client.GetClientTrace(commandName+" - http get", commandName, ruid, &tn)
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
transport := http.DefaultTransport
//transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
tn = time.Now()
res, err := transport.RoundTrip(req)
if err != nil {
log.Error(err.Error(), "ruid", ruid)
return err
}
log.Info("http get response", "tuid", tuid, "ruid", ruid, "endpoint", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength)
if res.StatusCode != 200 {
err := fmt.Errorf("expected status code %d, got %v", 200, res.StatusCode)
log.Warn(err.Error(), "ruid", ruid)
return err
}
defer res.Body.Close()
rdigest, err := digest(res.Body)
if err != nil {
log.Warn(err.Error(), "ruid", ruid)
return err
}
if !bytes.Equal(rdigest, original) {
err := fmt.Errorf("downloaded imported file md5=%x is not the same as the generated one=%x", rdigest, original)
log.Warn(err.Error(), "ruid", ruid)
return err
}
log.Trace("downloaded file matches random file", "ruid", ruid, "len", res.ContentLength)
return nil
}
// upload an arbitrary byte as a plaintext file to `endpoint` using the api client
func upload(data []byte, endpoint string) (string, error) {
swarm := client.NewClient(endpoint)
f := &client.File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
ManifestEntry: api.ManifestEntry{
ContentType: "text/plain",
Mode: 0660,
Size: int64(len(data)),
},
}
// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
return swarm.Upload(f, "", false)
}
func digest(r io.Reader) ([]byte, error) {
h := md5.New()
_, err := io.Copy(h, r)
if err != nil {
return nil, err
}
return h.Sum(nil), nil
}
// generates random data in heap buffer
func generateRandomData(datasize int) ([]byte, error) {
b := make([]byte, datasize)
c, err := crand.Read(b)
if err != nil {
return nil, err
} else if c != datasize {
return nil, errors.New("short read")
}
return b, nil
}

157
cmd/swarm/swarm-snapshot/create.go
View file

@ -1,157 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/node"
"github.com/ubiq/go-ubiq/p2p/simulations"
"github.com/ubiq/go-ubiq/p2p/simulations/adapters"
"github.com/ubiq/go-ubiq/swarm/network"
"github.com/ubiq/go-ubiq/swarm/network/simulation"
cli "gopkg.in/urfave/cli.v1"
)
// create is used as the entry function for "create" app command.
func create(ctx *cli.Context) error {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true))))
if len(ctx.Args()) < 1 {
return errors.New("argument should be the filename to verify or write-to")
}
filename, err := touchPath(ctx.Args()[0])
if err != nil {
return err
}
return createSnapshot(filename, ctx.Int("nodes"), strings.Split(ctx.String("services"), ","))
}
// createSnapshot creates a new snapshot on filesystem with provided filename,
// number of nodes and service names.
func createSnapshot(filename string, nodes int, services []string) (err error) {
log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services)
sim := simulation.New(map[string]simulation.ServiceFunc{
"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
addr := network.NewAddr(ctx.Config.Node())
kad := network.NewKademlia(addr.Over(), network.NewKadParams())
hp := network.NewHiveParams()
hp.KeepAliveInterval = time.Duration(200) * time.Millisecond
hp.Discovery = true // discovery must be enabled when creating a snapshot
config := &network.BzzConfig{
OverlayAddr: addr.Over(),
UnderlayAddr: addr.Under(),
HiveParams: hp,
}
return network.NewBzz(config, kad, nil, nil, nil), nil, nil
},
})
defer sim.Close()
_, err = sim.AddNodes(nodes)
if err != nil {
return fmt.Errorf("add nodes: %v", err)
}
err = sim.Net.ConnectNodesRing(nil)
if err != nil {
return fmt.Errorf("connect nodes: %v", err)
}
ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancelSimRun()
if _, err := sim.WaitTillHealthy(ctx); err != nil {
return fmt.Errorf("wait for healthy kademlia: %v", err)
}
var snap *simulations.Snapshot
if len(services) > 0 {
// If service names are provided, include them in the snapshot.
// But, check if "bzz" service is not among them to remove it
// form the snapshot as it exists on snapshot creation.
var removeServices []string
var wantBzz bool
for _, s := range services {
if s == "bzz" {
wantBzz = true
break
}
}
if !wantBzz {
removeServices = []string{"bzz"}
}
snap, err = sim.Net.SnapshotWithServices(services, removeServices)
} else {
snap, err = sim.Net.Snapshot()
}
if err != nil {
return fmt.Errorf("create snapshot: %v", err)
}
jsonsnapshot, err := json.Marshal(snap)
if err != nil {
return fmt.Errorf("json encode snapshot: %v", err)
}
return ioutil.WriteFile(filename, jsonsnapshot, 0666)
}
// touchPath creates an empty file and all subdirectories
// that are missing.
func touchPath(filename string) (string, error) {
if path.IsAbs(filename) {
if _, err := os.Stat(filename); err == nil {
// path exists, overwrite
return filename, nil
}
}
d, f := path.Split(filename)
dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
if err != nil {
return "", err
}
_, err = os.Stat(path.Join(dir, filename))
if err == nil {
// path exists, overwrite
return filename, nil
}
dirPath := path.Join(dir, d)
filePath := path.Join(dirPath, f)
if d != "" {
err = os.MkdirAll(dirPath, os.ModeDir)
if err != nil {
return "", err
}
}
return filePath, nil
}

143
cmd/swarm/swarm-snapshot/create_test.go
View file

@ -1,143 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"runtime"
"sort"
"strconv"
"strings"
"testing"
"github.com/ubiq/go-ubiq/p2p/simulations"
)
// TestSnapshotCreate is a high level e2e test that tests for snapshot generation.
// It runs a few "create" commands with different flag values and loads generated
// snapshot files to validate their content.
func TestSnapshotCreate(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
for _, v := range []struct {
name string
nodes int
services string
}{
{
name: "defaults",
},
{
name: "more nodes",
nodes: defaultNodes + 5,
},
{
name: "services",
services: "stream,pss,zorglub",
},
{
name: "services with bzz",
services: "bzz,pss",
},
} {
t.Run(v.name, func(t *testing.T) {
t.Parallel()
file, err := ioutil.TempFile("", "swarm-snapshot")
if err != nil {
t.Fatal(err)
}
defer os.Remove(file.Name())
if err = file.Close(); err != nil {
t.Error(err)
}
args := []string{"create"}
if v.nodes > 0 {
args = append(args, "--nodes", strconv.Itoa(v.nodes))
}
if v.services != "" {
args = append(args, "--services", v.services)
}
testCmd := runSnapshot(t, append(args, file.Name())...)
testCmd.ExpectExit()
if code := testCmd.ExitStatus(); code != 0 {
t.Fatalf("command exit code %v, expected 0", code)
}
f, err := os.Open(file.Name())
if err != nil {
t.Fatal(err)
}
defer func() {
err := f.Close()
if err != nil {
t.Error("closing snapshot file", "err", err)
}
}()
b, err := ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}
var snap simulations.Snapshot
err = json.Unmarshal(b, &snap)
if err != nil {
t.Fatal(err)
}
wantNodes := v.nodes
if wantNodes == 0 {
wantNodes = defaultNodes
}
gotNodes := len(snap.Nodes)
if gotNodes != wantNodes {
t.Errorf("got %v nodes, want %v", gotNodes, wantNodes)
}
if len(snap.Conns) == 0 {
t.Error("no connections in a snapshot")
}
var wantServices []string
if v.services != "" {
wantServices = strings.Split(v.services, ",")
} else {
wantServices = []string{"bzz"}
}
// sort service names so they can be comparable
// as strings to every node sorted services
sort.Strings(wantServices)
for i, n := range snap.Nodes {
gotServices := n.Node.Config.Services
sort.Strings(gotServices)
if fmt.Sprint(gotServices) != fmt.Sprint(wantServices) {
t.Errorf("got services %v for node %v, want %v", gotServices, i, wantServices)
}
}
})
}
}

82
cmd/swarm/swarm-snapshot/main.go
View file

@ -1,82 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"os"
"github.com/ubiq/go-ubiq/cmd/utils"
"github.com/ubiq/go-ubiq/log"
cli "gopkg.in/urfave/cli.v1"
)
var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
// default value for "create" command --nodes flag
const defaultNodes = 10
func main() {
err := newApp().Run(os.Args)
if err != nil {
log.Error(err.Error())
os.Exit(1)
}
}
// newApp construct a new instance of Swarm Snapshot Utility.
// Method Run is called on it in the main function and in tests.
func newApp() (app *cli.App) {
app = utils.NewApp(gitCommit, "Swarm Snapshot Utility")
app.Name = "swarm-snapshot"
app.Usage = ""
// app flags (for all commands)
app.Flags = []cli.Flag{
cli.IntFlag{
Name: "verbosity",
Value: 1,
Usage: "verbosity level",
},
}
app.Commands = []cli.Command{
{
Name: "create",
Aliases: []string{"c"},
Usage: "create a swarm snapshot",
Action: create,
// Flags only for "create" command.
// Allow app flags to be specified after the
// command argument.
Flags: append(app.Flags,
cli.IntFlag{
Name: "nodes",
Value: defaultNodes,
Usage: "number of nodes",
},
cli.StringFlag{
Name: "services",
Value: "bzz",
Usage: "comma separated list of services to boot the nodes with",
},
),
},
}
return app
}

49
cmd/swarm/swarm-snapshot/run_test.go
View file

@ -1,49 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"fmt"
"os"
"testing"
"github.com/docker/docker/pkg/reexec"
"github.com/ubiq/go-ubiq/internal/cmdtest"
)
func init() {
reexec.Register("swarm-snapshot", func() {
if err := newApp().Run(os.Args); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
os.Exit(0)
})
}
func runSnapshot(t *testing.T, args ...string) *cmdtest.TestCmd {
tt := cmdtest.NewTestCmd(t, nil)
tt.Run("swarm-snapshot", args...)
return tt
}
func TestMain(m *testing.M) {
if reexec.Init() {
return
}
os.Exit(m.Run())
}

188
cmd/swarm/upload.go
View file

@ -1,188 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
// Command bzzup uploads files to the swarm HTTP API.
package main
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"os/user"
"path"
"path/filepath"
"strconv"
"strings"
"github.com/ubiq/go-ubiq/log"
swarm "github.com/ubiq/go-ubiq/swarm/api/client"
"github.com/ubiq/go-ubiq/cmd/utils"
"gopkg.in/urfave/cli.v1"
)
var upCommand = cli.Command{
Action: upload,
CustomHelpTemplate: helpTemplate,
Name: "up",
Usage: "uploads a file or directory to swarm using the HTTP API",
ArgsUsage: "<file>",
Flags: []cli.Flag{SwarmEncryptedFlag},
Description: "uploads a file or directory to swarm using the HTTP API and prints the root hash",
}
func upload(ctx *cli.Context) {
args := ctx.Args()
var (
bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
recursive = ctx.GlobalBool(SwarmRecursiveFlag.Name)
wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
defaultPath = ctx.GlobalString(SwarmUploadDefaultPath.Name)
fromStdin = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
mimeType = ctx.GlobalString(SwarmUploadMimeType.Name)
client = swarm.NewClient(bzzapi)
toEncrypt = ctx.Bool(SwarmEncryptedFlag.Name)
autoDefaultPath = false
file string
)
if autoDefaultPathString := os.Getenv(SWARM_AUTO_DEFAULTPATH); autoDefaultPathString != "" {
b, err := strconv.ParseBool(autoDefaultPathString)
if err != nil {
utils.Fatalf("invalid environment variable %s: %v", SWARM_AUTO_DEFAULTPATH, err)
}
autoDefaultPath = b
}
if len(args) != 1 {
if fromStdin {
tmp, err := ioutil.TempFile("", "swarm-stdin")
if err != nil {
utils.Fatalf("error create tempfile: %s", err)
}
defer os.Remove(tmp.Name())
n, err := io.Copy(tmp, os.Stdin)
if err != nil {
utils.Fatalf("error copying stdin to tempfile: %s", err)
} else if n == 0 {
utils.Fatalf("error reading from stdin: zero length")
}
file = tmp.Name()
} else {
utils.Fatalf("Need filename as the first and only argument")
}
} else {
file = expandPath(args[0])
}
if !wantManifest {
f, err := swarm.Open(file)
if err != nil {
utils.Fatalf("Error opening file: %s", err)
}
defer f.Close()
hash, err := client.UploadRaw(f, f.Size, toEncrypt)
if err != nil {
utils.Fatalf("Upload failed: %s", err)
}
fmt.Println(hash)
return
}
stat, err := os.Stat(file)
if err != nil {
utils.Fatalf("Error opening file: %s", err)
}
// define a function which either uploads a directory or single file
// based on the type of the file being uploaded
var doUpload func() (hash string, err error)
if stat.IsDir() {
doUpload = func() (string, error) {
if !recursive {
return "", errors.New("Argument is a directory and recursive upload is disabled")
}
if autoDefaultPath && defaultPath == "" {
defaultEntryCandidate := path.Join(file, "index.html")
log.Debug("trying to find default path", "path", defaultEntryCandidate)
defaultEntryStat, err := os.Stat(defaultEntryCandidate)
if err == nil && !defaultEntryStat.IsDir() {
log.Debug("setting auto detected default path", "path", defaultEntryCandidate)
defaultPath = defaultEntryCandidate
}
}
if defaultPath != "" {
// construct absolute default path
absDefaultPath, _ := filepath.Abs(defaultPath)
absFile, _ := filepath.Abs(file)
// make sure absolute directory ends with only one "/"
// to trim it from absolute default path and get relative default path
absFile = strings.TrimRight(absFile, "/") + "/"
if absDefaultPath != "" && absFile != "" && strings.HasPrefix(absDefaultPath, absFile) {
defaultPath = strings.TrimPrefix(absDefaultPath, absFile)
}
}
return client.UploadDirectory(file, defaultPath, "", toEncrypt)
}
} else {
doUpload = func() (string, error) {
f, err := swarm.Open(file)
if err != nil {
return "", fmt.Errorf("error opening file: %s", err)
}
defer f.Close()
if mimeType != "" {
f.ContentType = mimeType
}
return client.Upload(f, "", toEncrypt)
}
}
hash, err := doUpload()
if err != nil {
utils.Fatalf("Upload failed: %s", err)
}
fmt.Println(hash)
}
// Expands a file path
// 1. replace tilde with users home dir
// 2. expands embedded environment variables
// 3. cleans the path, e.g. /a/b/../c -> /a/c
// Note, it has limitations, e.g. ~someuser/tmp will not be expanded
func expandPath(p string) string {
if i := strings.Index(p, ":"); i > 0 {
return p
}
if i := strings.Index(p, "@"); i > 0 {
return p
}
if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") {
if home := homeDir(); home != "" {
p = home + p[1:]
}
}
return path.Clean(os.ExpandEnv(p))
}
func homeDir() string {
if home := os.Getenv("HOME"); home != "" {
return home
}
if usr, err := user.Current(); err == nil {
return usr.HomeDir
}
return ""
}

356
cmd/swarm/upload_test.go
View file

@ -1,356 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/ubiq/go-ubiq/log"
swarmapi "github.com/ubiq/go-ubiq/swarm/api/client"
"github.com/ubiq/go-ubiq/swarm/testutil"
"github.com/mattn/go-colorable"
)
func init() {
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
}
func TestSwarmUp(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip()
}
initCluster(t)
cases := []struct {
name string
f func(t *testing.T)
}{
{"NoEncryption", testNoEncryption},
{"Encrypted", testEncrypted},
{"RecursiveNoEncryption", testRecursiveNoEncryption},
{"RecursiveEncrypted", testRecursiveEncrypted},
{"DefaultPathAll", testDefaultPathAll},
}
for _, tc := range cases {
t.Run(tc.name, tc.f)
}
}
// testNoEncryption tests that running 'swarm up' makes the resulting file
// available from all nodes via the HTTP API
func testNoEncryption(t *testing.T) {
testDefault(false, t)
}
// testEncrypted tests that running 'swarm up --encrypted' makes the resulting file
// available from all nodes via the HTTP API
func testEncrypted(t *testing.T) {
testDefault(true, t)
}
func testRecursiveNoEncryption(t *testing.T) {
testRecursive(false, t)
}
func testRecursiveEncrypted(t *testing.T) {
testRecursive(true, t)
}
func testDefault(toEncrypt bool, t *testing.T) {
tmpFileName := testutil.TempFileWithContent(t, data)
defer os.Remove(tmpFileName)
// write data to file
hashRegexp := `[a-f\d]{64}`
flags := []string{
"--bzzapi", cluster.Nodes[0].URL,
"up",
tmpFileName}
if toEncrypt {
hashRegexp = `[a-f\d]{128}`
flags = []string{
"--bzzapi", cluster.Nodes[0].URL,
"up",
"--encrypt",
tmpFileName}
}
// upload the file with 'swarm up' and expect a hash
log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
up := runSwarm(t, flags...)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
hash := matches[0]
log.Info("file uploaded", "hash", hash)
// get the file from the HTTP API of each node
for _, node := range cluster.Nodes {
log.Info("getting file from node", "node", node.Name)
res, err := http.Get(node.URL + "/bzz:/" + hash)
if err != nil {
t.Fatal(err)
}
defer res.Body.Close()
reply, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Fatal(err)
}
if res.StatusCode != 200 {
t.Fatalf("expected HTTP status 200, got %s", res.Status)
}
if string(reply) != data {
t.Fatalf("expected HTTP body %q, got %q", data, reply)
}
log.Debug("verifying uploaded file using `swarm down`")
//try to get the content with `swarm down`
tmpDownload, err := ioutil.TempDir("", "swarm-test")
tmpDownload = path.Join(tmpDownload, "tmpfile.tmp")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDownload)
bzzLocator := "bzz:/" + hash
flags = []string{
"--bzzapi", cluster.Nodes[0].URL,
"down",
bzzLocator,
tmpDownload,
}
down := runSwarm(t, flags...)
down.ExpectExit()
fi, err := os.Stat(tmpDownload)
if err != nil {
t.Fatalf("could not stat path: %v", err)
}
switch mode := fi.Mode(); {
case mode.IsRegular():
downloadedBytes, err := ioutil.ReadFile(tmpDownload)
if err != nil {
t.Fatalf("had an error reading the downloaded file: %v", err)
}
if !bytes.Equal(downloadedBytes, bytes.NewBufferString(data).Bytes()) {
t.Fatalf("retrieved data and posted data not equal!")
}
default:
t.Fatalf("expected to download regular file, got %s", fi.Mode())
}
}
timeout := time.Duration(2 * time.Second)
httpClient := http.Client{
Timeout: timeout,
}
// try to squeeze a timeout by getting an non-existent hash from each node
for _, node := range cluster.Nodes {
_, err := httpClient.Get(node.URL + "/bzz:/1023e8bae0f70be7d7b5f74343088ba408a218254391490c85ae16278e230340")
// we're speeding up the timeout here since netstore has a 60 seconds timeout on a request
if err != nil && !strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") {
t.Fatal(err)
}
// this is disabled since it takes 60s due to netstore timeout
// if res.StatusCode != 404 {
// t.Fatalf("expected HTTP status 404, got %s", res.Status)
// }
}
}
func testRecursive(toEncrypt bool, t *testing.T) {
tmpUploadDir, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpUploadDir)
// create tmp files
for _, path := range []string{"tmp1", "tmp2"} {
if err := ioutil.WriteFile(filepath.Join(tmpUploadDir, path), bytes.NewBufferString(data).Bytes(), 0644); err != nil {
t.Fatal(err)
}
}
hashRegexp := `[a-f\d]{64}`
flags := []string{
"--bzzapi", cluster.Nodes[0].URL,
"--recursive",
"up",
tmpUploadDir}
if toEncrypt {
hashRegexp = `[a-f\d]{128}`
flags = []string{
"--bzzapi", cluster.Nodes[0].URL,
"--recursive",
"up",
"--encrypt",
tmpUploadDir}
}
// upload the file with 'swarm up' and expect a hash
log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
up := runSwarm(t, flags...)
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
hash := matches[0]
log.Info("dir uploaded", "hash", hash)
// get the file from the HTTP API of each node
for _, node := range cluster.Nodes {
log.Info("getting file from node", "node", node.Name)
//try to get the content with `swarm down`
tmpDownload, err := ioutil.TempDir("", "swarm-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDownload)
bzzLocator := "bzz:/" + hash
flagss := []string{
"--bzzapi", cluster.Nodes[0].URL,
"down",
"--recursive",
bzzLocator,
tmpDownload,
}
fmt.Println("downloading from swarm with recursive")
down := runSwarm(t, flagss...)
down.ExpectExit()
files, err := ioutil.ReadDir(tmpDownload)
for _, v := range files {
fi, err := os.Stat(path.Join(tmpDownload, v.Name()))
if err != nil {
t.Fatalf("got an error: %v", err)
}
switch mode := fi.Mode(); {
case mode.IsRegular():
if file, err := swarmapi.Open(path.Join(tmpDownload, v.Name())); err != nil {
t.Fatalf("encountered an error opening the file returned from the CLI: %v", err)
} else {
ff := make([]byte, len(data))
io.ReadFull(file, ff)
buf := bytes.NewBufferString(data)
if !bytes.Equal(ff, buf.Bytes()) {
t.Fatalf("retrieved data and posted data not equal!")
}
}
default:
t.Fatalf("this shouldnt happen")
}
}
if err != nil {
t.Fatalf("could not list files at: %v", files)
}
}
}
// testDefaultPathAll tests swarm recursive upload with relative and absolute
// default paths and with encryption.
func testDefaultPathAll(t *testing.T) {
testDefaultPath(false, false, t)
testDefaultPath(false, true, t)
testDefaultPath(true, false, t)
testDefaultPath(true, true, t)
}
func testDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) {
tmp, err := ioutil.TempDir("", "swarm-defaultpath-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
err = ioutil.WriteFile(filepath.Join(tmp, "index.html"), []byte("<h1>Test</h1>"), 0666)
if err != nil {
t.Fatal(err)
}
err = ioutil.WriteFile(filepath.Join(tmp, "robots.txt"), []byte("Disallow: /"), 0666)
if err != nil {
t.Fatal(err)
}
defaultPath := "index.html"
if absDefaultPath {
defaultPath = filepath.Join(tmp, defaultPath)
}
args := []string{
"--bzzapi",
cluster.Nodes[0].URL,
"--recursive",
"--defaultpath",
defaultPath,
"up",
tmp,
}
if toEncrypt {
args = append(args, "--encrypt")
}
up := runSwarm(t, args...)
hashRegexp := `[a-f\d]{64,128}`
_, matches := up.ExpectRegexp(hashRegexp)
up.ExpectExit()
hash := matches[0]
client := swarmapi.NewClient(cluster.Nodes[0].URL)
m, isEncrypted, err := client.DownloadManifest(hash)
if err != nil {
t.Fatal(err)
}
if toEncrypt != isEncrypted {
t.Error("downloaded manifest is not encrypted")
}
var found bool
var entriesCount int
for _, e := range m.Entries {
entriesCount++
if e.Path == "" {
found = true
}
}
if !found {
t.Error("manifest default entry was not found")
}
if entriesCount != 3 {
t.Errorf("manifest contains %v entries, expected %v", entriesCount, 3)
}
}

68
contracts/chequebook/api.go
View file

@ -1,68 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package chequebook
import (
"errors"
"math/big"
"github.com/ubiq/go-ubiq/common"
)
const Version = "1.0"
var errNoChequebook = errors.New("no chequebook")
type API struct {
chequebookf func() *Chequebook
}
func NewAPI(ch func() *Chequebook) *API {
return &API{ch}
}
func (a *API) Balance() (string, error) {
ch := a.chequebookf()
if ch == nil {
return "", errNoChequebook
}
return ch.Balance().String(), nil
}
func (a *API) Issue(beneficiary common.Address, amount *big.Int) (cheque *Cheque, err error) {
ch := a.chequebookf()
if ch == nil {
return nil, errNoChequebook
}
return ch.Issue(beneficiary, amount)
}
func (a *API) Cash(cheque *Cheque) (txhash string, err error) {
ch := a.chequebookf()
if ch == nil {
return "", errNoChequebook
}
return ch.Cash(cheque)
}
func (a *API) Deposit(amount *big.Int) (txhash string, err error) {
ch := a.chequebookf()
if ch == nil {
return "", errNoChequebook
}
return ch.Deposit(amount)
}

641
contracts/chequebook/cheque.go
View file

@ -1,641 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package chequebook package wraps the 'chequebook' Ethereum smart contract.
//
// The functions in this package allow using chequebook for
// issuing, receiving, verifying cheques in ether; (auto)cashing cheques in ether
// as well as (auto)depositing ether to the chequebook contract.
package chequebook
//go:generate abigen --sol contract/chequebook.sol --exc contract/mortal.sol:mortal,contract/owned.sol:owned --pkg contract --out contract/chequebook.go
//go:generate go run ./gencode.go
import (
"bytes"
"context"
"crypto/ecdsa"
"encoding/json"
"fmt"
"io/ioutil"
"math/big"
"os"
"sync"
"time"
"github.com/ubiq/go-ubiq/accounts/abi/bind"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/common/hexutil"
"github.com/ubiq/go-ubiq/contracts/chequebook/contract"
"github.com/ubiq/go-ubiq/core/types"
"github.com/ubiq/go-ubiq/crypto"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/swarm/services/swap/swap"
)
// TODO(zelig): watch peer solvency and notify of bouncing cheques
// TODO(zelig): enable paying with cheque by signing off
// Some functionality requires interacting with the blockchain:
// * setting current balance on peer's chequebook
// * sending the transaction to cash the cheque
// * depositing ether to the chequebook
// * watching incoming ether
var (
gasToCash = uint64(2000000) // gas cost of a cash transaction using chequebook
// gasToDeploy = uint64(3000000)
)
// Backend wraps all methods required for chequebook operation.
type Backend interface {
bind.ContractBackend
TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error)
BalanceAt(ctx context.Context, address common.Address, blockNum *big.Int) (*big.Int, error)
}
// Cheque represents a payment promise to a single beneficiary.
type Cheque struct {
Contract common.Address // address of chequebook, needed to avoid cross-contract submission
Beneficiary common.Address
Amount *big.Int // cumulative amount of all funds sent
Sig []byte // signature Sign(Keccak256(contract, beneficiary, amount), prvKey)
}
func (ch *Cheque) String() string {
return fmt.Sprintf("contract: %s, beneficiary: %s, amount: %v, signature: %x", ch.Contract.Hex(), ch.Beneficiary.Hex(), ch.Amount, ch.Sig)
}
type Params struct {
ContractCode, ContractAbi string
}
var ContractParams = &Params{contract.ChequebookBin, contract.ChequebookABI}
// Chequebook can create and sign cheques from a single contract to multiple beneficiaries.
// It is the outgoing payment handler for peer to peer micropayments.
type Chequebook struct {
path string // path to chequebook file
prvKey *ecdsa.PrivateKey // private key to sign cheque with
lock sync.Mutex //
backend Backend // blockchain API
quit chan bool // when closed causes autodeposit to stop
owner common.Address // owner address (derived from pubkey)
contract *contract.Chequebook // abigen binding
session *contract.ChequebookSession // abigen binding with Tx Opts
// persisted fields
balance *big.Int // not synced with blockchain
contractAddr common.Address // contract address
sent map[common.Address]*big.Int //tallies for beneficiaries
txhash string // tx hash of last deposit tx
threshold *big.Int // threshold that triggers autodeposit if not nil
buffer *big.Int // buffer to keep on top of balance for fork protection
log log.Logger // contextual logger with the contract address embedded
}
func (chbook *Chequebook) String() string {
return fmt.Sprintf("contract: %s, owner: %s, balance: %v, signer: %x", chbook.contractAddr.Hex(), chbook.owner.Hex(), chbook.balance, chbook.prvKey.PublicKey)
}
// NewChequebook creates a new Chequebook.
func NewChequebook(path string, contractAddr common.Address, prvKey *ecdsa.PrivateKey, backend Backend) (self *Chequebook, err error) {
balance := new(big.Int)
sent := make(map[common.Address]*big.Int)
chbook, err := contract.NewChequebook(contractAddr, backend)
if err != nil {
return nil, err
}
transactOpts := bind.NewKeyedTransactor(prvKey)
session := &contract.ChequebookSession{
Contract: chbook,
TransactOpts: *transactOpts,
}
self = &Chequebook{
prvKey: prvKey,
balance: balance,
contractAddr: contractAddr,
sent: sent,
path: path,
backend: backend,
owner: transactOpts.From,
contract: chbook,
session: session,
log: log.New("contract", contractAddr),
}
if (contractAddr != common.Address{}) {
self.setBalanceFromBlockChain()
self.log.Trace("New chequebook initialised", "owner", self.owner, "balance", self.balance)
}
return
}
func (chbook *Chequebook) setBalanceFromBlockChain() {
balance, err := chbook.backend.BalanceAt(context.TODO(), chbook.contractAddr, nil)
if err != nil {
log.Error("Failed to retrieve chequebook balance", "err", err)
} else {
chbook.balance.Set(balance)
}
}
// LoadChequebook loads a chequebook from disk (file path).
func LoadChequebook(path string, prvKey *ecdsa.PrivateKey, backend Backend, checkBalance bool) (self *Chequebook, err error) {
var data []byte
data, err = ioutil.ReadFile(path)
if err != nil {
return
}
self, _ = NewChequebook(path, common.Address{}, prvKey, backend)
err = json.Unmarshal(data, self)
if err != nil {
return nil, err
}
if checkBalance {
self.setBalanceFromBlockChain()
}
log.Trace("Loaded chequebook from disk", "path", path)
return
}
// chequebookFile is the JSON representation of a chequebook.
type chequebookFile struct {
Balance string
Contract string
Owner string
Sent map[string]string
}
// UnmarshalJSON deserialises a chequebook.
func (chbook *Chequebook) UnmarshalJSON(data []byte) error {
var file chequebookFile
err := json.Unmarshal(data, &file)
if err != nil {
return err
}
_, ok := chbook.balance.SetString(file.Balance, 10)
if !ok {
return fmt.Errorf("cumulative amount sent: unable to convert string to big integer: %v", file.Balance)
}
chbook.contractAddr = common.HexToAddress(file.Contract)
for addr, sent := range file.Sent {
chbook.sent[common.HexToAddress(addr)], ok = new(big.Int).SetString(sent, 10)
if !ok {
return fmt.Errorf("beneficiary %v cumulative amount sent: unable to convert string to big integer: %v", addr, sent)
}
}
return nil
}
// MarshalJSON serialises a chequebook.
func (chbook *Chequebook) MarshalJSON() ([]byte, error) {
var file = &chequebookFile{
Balance: chbook.balance.String(),
Contract: chbook.contractAddr.Hex(),
Owner: chbook.owner.Hex(),
Sent: make(map[string]string),
}
for addr, sent := range chbook.sent {
file.Sent[addr.Hex()] = sent.String()
}
return json.Marshal(file)
}
// Save persists the chequebook on disk, remembering balance, contract address and
// cumulative amount of funds sent for each beneficiary.
func (chbook *Chequebook) Save() (err error) {
data, err := json.MarshalIndent(chbook, "", " ")
if err != nil {
return err
}
chbook.log.Trace("Saving chequebook to disk", chbook.path)
return ioutil.WriteFile(chbook.path, data, os.ModePerm)
}
// Stop quits the autodeposit go routine to terminate
func (chbook *Chequebook) Stop() {
defer chbook.lock.Unlock()
chbook.lock.Lock()
if chbook.quit != nil {
close(chbook.quit)
chbook.quit = nil
}
}
// Issue creates a cheque signed by the chequebook owner's private key. The
// signer commits to a contract (one that they own), a beneficiary and amount.
func (chbook *Chequebook) Issue(beneficiary common.Address, amount *big.Int) (ch *Cheque, err error) {
defer chbook.lock.Unlock()
chbook.lock.Lock()
if amount.Sign() <= 0 {
return nil, fmt.Errorf("amount must be greater than zero (%v)", amount)
}
if chbook.balance.Cmp(amount) < 0 {
err = fmt.Errorf("insufficient funds to issue cheque for amount: %v. balance: %v", amount, chbook.balance)
} else {
var sig []byte
sent, found := chbook.sent[beneficiary]
if !found {
sent = new(big.Int)
chbook.sent[beneficiary] = sent
}
sum := new(big.Int).Set(sent)
sum.Add(sum, amount)
sig, err = crypto.Sign(sigHash(chbook.contractAddr, beneficiary, sum), chbook.prvKey)
if err == nil {
ch = &Cheque{
Contract: chbook.contractAddr,
Beneficiary: beneficiary,
Amount: sum,
Sig: sig,
}
sent.Set(sum)
chbook.balance.Sub(chbook.balance, amount) // subtract amount from balance
}
}
// auto deposit if threshold is set and balance is less then threshold
// note this is called even if issuing cheque fails
// so we reattempt depositing
if chbook.threshold != nil {
if chbook.balance.Cmp(chbook.threshold) < 0 {
send := new(big.Int).Sub(chbook.buffer, chbook.balance)
chbook.deposit(send)
}
}
return
}
// Cash is a convenience method to cash any cheque.
func (chbook *Chequebook) Cash(ch *Cheque) (txhash string, err error) {
return ch.Cash(chbook.session)
}
// data to sign: contract address, beneficiary, cumulative amount of funds ever sent
func sigHash(contract, beneficiary common.Address, sum *big.Int) []byte {
bigamount := sum.Bytes()
if len(bigamount) > 32 {
return nil
}
var amount32 [32]byte
copy(amount32[32-len(bigamount):32], bigamount)
input := append(contract.Bytes(), beneficiary.Bytes()...)
input = append(input, amount32[:]...)
return crypto.Keccak256(input)
}
// Balance returns the current balance of the chequebook.
func (chbook *Chequebook) Balance() *big.Int {
defer chbook.lock.Unlock()
chbook.lock.Lock()
return new(big.Int).Set(chbook.balance)
}
// Owner returns the owner account of the chequebook.
func (chbook *Chequebook) Owner() common.Address {
return chbook.owner
}
// Address returns the on-chain contract address of the chequebook.
func (chbook *Chequebook) Address() common.Address {
return chbook.contractAddr
}
// Deposit deposits money to the chequebook account.
func (chbook *Chequebook) Deposit(amount *big.Int) (string, error) {
defer chbook.lock.Unlock()
chbook.lock.Lock()
return chbook.deposit(amount)
}
// deposit deposits amount to the chequebook account.
// The caller must hold self.lock.
func (chbook *Chequebook) deposit(amount *big.Int) (string, error) {
// since the amount is variable here, we do not use sessions
depositTransactor := bind.NewKeyedTransactor(chbook.prvKey)
depositTransactor.Value = amount
chbookRaw := &contract.ChequebookRaw{Contract: chbook.contract}
tx, err := chbookRaw.Transfer(depositTransactor)
if err != nil {
chbook.log.Warn("Failed to fund chequebook", "amount", amount, "balance", chbook.balance, "target", chbook.buffer, "err", err)
return "", err
}
// assume that transaction is actually successful, we add the amount to balance right away
chbook.balance.Add(chbook.balance, amount)
chbook.log.Trace("Deposited funds to chequebook", "amount", amount, "balance", chbook.balance, "target", chbook.buffer)
return tx.Hash().Hex(), nil
}
// AutoDeposit (re)sets interval time and amount which triggers sending funds to the
// chequebook. Contract backend needs to be set if threshold is not less than buffer, then
// deposit will be triggered on every new cheque issued.
func (chbook *Chequebook) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
defer chbook.lock.Unlock()
chbook.lock.Lock()
chbook.threshold = threshold
chbook.buffer = buffer
chbook.autoDeposit(interval)
}
// autoDeposit starts a goroutine that periodically sends funds to the chequebook
// contract caller holds the lock the go routine terminates if Chequebook.quit is closed.
func (chbook *Chequebook) autoDeposit(interval time.Duration) {
if chbook.quit != nil {
close(chbook.quit)
chbook.quit = nil
}
// if threshold >= balance autodeposit after every cheque issued
if interval == time.Duration(0) || chbook.threshold != nil && chbook.buffer != nil && chbook.threshold.Cmp(chbook.buffer) >= 0 {
return
}
ticker := time.NewTicker(interval)
chbook.quit = make(chan bool)
quit := chbook.quit
go func() {
for {
select {
case <-quit:
return
case <-ticker.C:
chbook.lock.Lock()
if chbook.balance.Cmp(chbook.buffer) < 0 {
amount := new(big.Int).Sub(chbook.buffer, chbook.balance)
txhash, err := chbook.deposit(amount)
if err == nil {
chbook.txhash = txhash
}
}
chbook.lock.Unlock()
}
}
}()
}
// Outbox can issue cheques from a single contract to a single beneficiary.
type Outbox struct {
chequeBook *Chequebook
beneficiary common.Address
}
// NewOutbox creates an outbox.
func NewOutbox(chbook *Chequebook, beneficiary common.Address) *Outbox {
return &Outbox{chbook, beneficiary}
}
// Issue creates cheque.
func (o *Outbox) Issue(amount *big.Int) (swap.Promise, error) {
return o.chequeBook.Issue(o.beneficiary, amount)
}
// AutoDeposit enables auto-deposits on the underlying chequebook.
func (o *Outbox) AutoDeposit(interval time.Duration, threshold, buffer *big.Int) {
o.chequeBook.AutoDeposit(interval, threshold, buffer)
}
// Stop helps satisfy the swap.OutPayment interface.
func (o *Outbox) Stop() {}
// String implements fmt.Stringer.
func (o *Outbox) String() string {
return fmt.Sprintf("chequebook: %v, beneficiary: %s, balance: %v", o.chequeBook.Address().Hex(), o.beneficiary.Hex(), o.chequeBook.Balance())
}
// Inbox can deposit, verify and cash cheques from a single contract to a single
// beneficiary. It is the incoming payment handler for peer to peer micropayments.
type Inbox struct {
lock sync.Mutex
contract common.Address // peer's chequebook contract
beneficiary common.Address // local peer's receiving address
sender common.Address // local peer's address to send cashing tx from
signer *ecdsa.PublicKey // peer's public key
txhash string // tx hash of last cashing tx
session *contract.ChequebookSession // abi contract backend with tx opts
quit chan bool // when closed causes autocash to stop
maxUncashed *big.Int // threshold that triggers autocashing
cashed *big.Int // cumulative amount cashed
cheque *Cheque // last cheque, nil if none yet received
log log.Logger // contextual logger with the contract address embedded
}
// NewInbox creates an Inbox. An Inboxes is not persisted, the cumulative sum is updated
// from blockchain when first cheque is received.
func NewInbox(prvKey *ecdsa.PrivateKey, contractAddr, beneficiary common.Address, signer *ecdsa.PublicKey, abigen bind.ContractBackend) (self *Inbox, err error) {
if signer == nil {
return nil, fmt.Errorf("signer is null")
}
chbook, err := contract.NewChequebook(contractAddr, abigen)
if err != nil {
return nil, err
}
transactOpts := bind.NewKeyedTransactor(prvKey)
transactOpts.GasLimit = gasToCash
session := &contract.ChequebookSession{
Contract: chbook,
TransactOpts: *transactOpts,
}
sender := transactOpts.From
self = &Inbox{
contract: contractAddr,
beneficiary: beneficiary,
sender: sender,
signer: signer,
session: session,
cashed: new(big.Int).Set(common.Big0),
log: log.New("contract", contractAddr),
}
self.log.Trace("New chequebook inbox initialized", "beneficiary", self.beneficiary, "signer", hexutil.Bytes(crypto.FromECDSAPub(signer)))
return
}
func (i *Inbox) String() string {
return fmt.Sprintf("chequebook: %v, beneficiary: %s, balance: %v", i.contract.Hex(), i.beneficiary.Hex(), i.cheque.Amount)
}
// Stop quits the autocash goroutine.
func (i *Inbox) Stop() {
defer i.lock.Unlock()
i.lock.Lock()
if i.quit != nil {
close(i.quit)
i.quit = nil
}
}
// Cash attempts to cash the current cheque.
func (i *Inbox) Cash() (txhash string, err error) {
if i.cheque != nil {
txhash, err = i.cheque.Cash(i.session)
i.log.Trace("Cashing in chequebook cheque", "amount", i.cheque.Amount, "beneficiary", i.beneficiary)
i.cashed = i.cheque.Amount
}
return
}
// AutoCash (re)sets maximum time and amount which triggers cashing of the last uncashed
// cheque if maxUncashed is set to 0, then autocash on receipt.
func (i *Inbox) AutoCash(cashInterval time.Duration, maxUncashed *big.Int) {
defer i.lock.Unlock()
i.lock.Lock()
i.maxUncashed = maxUncashed
i.autoCash(cashInterval)
}
// autoCash starts a loop that periodically clears the last cheque
// if the peer is trusted. Clearing period could be 24h or a week.
// The caller must hold self.lock.
func (i *Inbox) autoCash(cashInterval time.Duration) {
if i.quit != nil {
close(i.quit)
i.quit = nil
}
// if maxUncashed is set to 0, then autocash on receipt
if cashInterval == time.Duration(0) || i.maxUncashed != nil && i.maxUncashed.Sign() == 0 {
return
}
ticker := time.NewTicker(cashInterval)
i.quit = make(chan bool)
quit := i.quit
go func() {
for {
select {
case <-quit:
return
case <-ticker.C:
i.lock.Lock()
if i.cheque != nil && i.cheque.Amount.Cmp(i.cashed) != 0 {
txhash, err := i.Cash()
if err == nil {
i.txhash = txhash
}
}
i.lock.Unlock()
}
}
}()
}
// Receive is called to deposit the latest cheque to the incoming Inbox.
// The given promise must be a *Cheque.
func (i *Inbox) Receive(promise swap.Promise) (*big.Int, error) {
ch := promise.(*Cheque)
defer i.lock.Unlock()
i.lock.Lock()
var sum *big.Int
if i.cheque == nil {
// the sum is checked against the blockchain once a cheque is received
tally, err := i.session.Sent(i.beneficiary)
if err != nil {
return nil, fmt.Errorf("inbox: error calling backend to set amount: %v", err)
}
sum = tally
} else {
sum = i.cheque.Amount
}
amount, err := ch.Verify(i.signer, i.contract, i.beneficiary, sum)
var uncashed *big.Int
if err == nil {
i.cheque = ch
if i.maxUncashed != nil {
uncashed = new(big.Int).Sub(ch.Amount, i.cashed)
if i.maxUncashed.Cmp(uncashed) < 0 {
i.Cash()
}
}
i.log.Trace("Received cheque in chequebook inbox", "amount", amount, "uncashed", uncashed)
}
return amount, err
}
// Verify verifies cheque for signer, contract, beneficiary, amount, valid signature.
func (ch *Cheque) Verify(signerKey *ecdsa.PublicKey, contract, beneficiary common.Address, sum *big.Int) (*big.Int, error) {
log.Trace("Verifying chequebook cheque", "cheque", ch, "sum", sum)
if sum == nil {
return nil, fmt.Errorf("invalid amount")
}
if ch.Beneficiary != beneficiary {
return nil, fmt.Errorf("beneficiary mismatch: %v != %v", ch.Beneficiary.Hex(), beneficiary.Hex())
}
if ch.Contract != contract {
return nil, fmt.Errorf("contract mismatch: %v != %v", ch.Contract.Hex(), contract.Hex())
}
amount := new(big.Int).Set(ch.Amount)
if sum != nil {
amount.Sub(amount, sum)
if amount.Sign() <= 0 {
return nil, fmt.Errorf("incorrect amount: %v <= 0", amount)
}
}
pubKey, err := crypto.SigToPub(sigHash(ch.Contract, beneficiary, ch.Amount), ch.Sig)
if err != nil {
return nil, fmt.Errorf("invalid signature: %v", err)
}
if !bytes.Equal(crypto.FromECDSAPub(pubKey), crypto.FromECDSAPub(signerKey)) {
return nil, fmt.Errorf("signer mismatch: %x != %x", crypto.FromECDSAPub(pubKey), crypto.FromECDSAPub(signerKey))
}
return amount, nil
}
// v/r/s representation of signature
func sig2vrs(sig []byte) (v byte, r, s [32]byte) {
v = sig[64] + 27
copy(r[:], sig[:32])
copy(s[:], sig[32:64])
return
}
// Cash cashes the cheque by sending an Ethereum transaction.
func (ch *Cheque) Cash(session *contract.ChequebookSession) (string, error) {
v, r, s := sig2vrs(ch.Sig)
tx, err := session.Cash(ch.Beneficiary, ch.Amount, v, r, s)
if err != nil {
return "", err
}
return tx.Hash().Hex(), nil
}
// ValidateCode checks that the on-chain code at address matches the expected chequebook
// contract code. This is used to detect suicided chequebooks.
func ValidateCode(ctx context.Context, b Backend, address common.Address) (ok bool, err error) {
code, err := b.CodeAt(ctx, address, nil)
if err != nil {
return false, err
}
return bytes.Equal(code, common.FromHex(contract.ContractDeployedCode)), nil
}

487
contracts/chequebook/cheque_test.go
View file

@ -1,487 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package chequebook
import (
"crypto/ecdsa"
"math/big"
"os"
"path/filepath"
"testing"
"time"
"github.com/ubiq/go-ubiq/accounts/abi/bind"
"github.com/ubiq/go-ubiq/accounts/abi/bind/backends"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/contracts/chequebook/contract"
"github.com/ubiq/go-ubiq/core"
"github.com/ubiq/go-ubiq/crypto"
)
var (
key0, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
key1, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
key2, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee")
addr0 = crypto.PubkeyToAddress(key0.PublicKey)
addr1 = crypto.PubkeyToAddress(key1.PublicKey)
addr2 = crypto.PubkeyToAddress(key2.PublicKey)
)
func newTestBackend() *backends.SimulatedBackend {
return backends.NewSimulatedBackend(core.GenesisAlloc{
addr0: {Balance: big.NewInt(1000000000)},
addr1: {Balance: big.NewInt(1000000000)},
addr2: {Balance: big.NewInt(1000000000)},
}, 10000000)
}
func deploy(prvKey *ecdsa.PrivateKey, amount *big.Int, backend *backends.SimulatedBackend) (common.Address, error) {
deployTransactor := bind.NewKeyedTransactor(prvKey)
deployTransactor.Value = amount
addr, _, _, err := contract.DeployChequebook(deployTransactor, backend)
if err != nil {
return common.Address{}, err
}
backend.Commit()
return addr, nil
}
func TestIssueAndReceive(t *testing.T) {
path := filepath.Join(os.TempDir(), "chequebook-test.json")
backend := newTestBackend()
addr0, err := deploy(key0, big.NewInt(0), backend)
if err != nil {
t.Fatalf("deploy contract: expected no error, got %v", err)
}
chbook, err := NewChequebook(path, addr0, key0, backend)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
chbook.sent[addr1] = new(big.Int).SetUint64(42)
amount := common.Big1
if _, err = chbook.Issue(addr1, amount); err == nil {
t.Fatalf("expected insufficient funds error, got none")
}
chbook.balance = new(big.Int).Set(common.Big1)
if chbook.Balance().Cmp(common.Big1) != 0 {
t.Fatalf("expected: %v, got %v", "0", chbook.Balance())
}
ch, err := chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if chbook.Balance().Sign() != 0 {
t.Errorf("expected: %v, got %v", "0", chbook.Balance())
}
chbox, err := NewInbox(key1, addr0, addr1, &key0.PublicKey, backend)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
received, err := chbox.Receive(ch)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if received.Cmp(big.NewInt(43)) != 0 {
t.Errorf("expected: %v, got %v", "43", received)
}
}
func TestCheckbookFile(t *testing.T) {
path := filepath.Join(os.TempDir(), "chequebook-test.json")
backend := newTestBackend()
chbook, err := NewChequebook(path, addr0, key0, backend)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
chbook.sent[addr1] = new(big.Int).SetUint64(42)
chbook.balance = new(big.Int).Set(common.Big1)
chbook.Save()
chbook, err = LoadChequebook(path, key0, backend, false)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if chbook.Balance().Cmp(common.Big1) != 0 {
t.Errorf("expected: %v, got %v", "0", chbook.Balance())
}
var ch *Cheque
if ch, err = chbook.Issue(addr1, common.Big1); err != nil {
t.Fatalf("expected no error, got %v", err)
}
if ch.Amount.Cmp(new(big.Int).SetUint64(43)) != 0 {
t.Errorf("expected: %v, got %v", "0", ch.Amount)
}
err = chbook.Save()
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
}
func TestVerifyErrors(t *testing.T) {
path0 := filepath.Join(os.TempDir(), "chequebook-test-0.json")
backend := newTestBackend()
contr0, err := deploy(key0, common.Big2, backend)
if err != nil {
t.Errorf("expected no error, got %v", err)
}
chbook0, err := NewChequebook(path0, contr0, key0, backend)
if err != nil {
t.Errorf("expected no error, got %v", err)
}
path1 := filepath.Join(os.TempDir(), "chequebook-test-1.json")
contr1, _ := deploy(key1, common.Big2, backend)
chbook1, err := NewChequebook(path1, contr1, key1, backend)
if err != nil {
t.Errorf("expected no error, got %v", err)
}
chbook0.sent[addr1] = new(big.Int).SetUint64(42)
chbook0.balance = new(big.Int).Set(common.Big2)
chbook1.balance = new(big.Int).Set(common.Big1)
amount := common.Big1
ch0, err := chbook0.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
chbox, err := NewInbox(key1, contr0, addr1, &key0.PublicKey, backend)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
received, err := chbox.Receive(ch0)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if received.Cmp(big.NewInt(43)) != 0 {
t.Errorf("expected: %v, got %v", "43", received)
}
ch1, err := chbook0.Issue(addr2, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
received, err = chbox.Receive(ch1)
t.Logf("correct error: %v", err)
if err == nil {
t.Fatalf("expected receiver error, got none and value %v", received)
}
ch2, err := chbook1.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
received, err = chbox.Receive(ch2)
t.Logf("correct error: %v", err)
if err == nil {
t.Fatalf("expected sender error, got none and value %v", received)
}
_, err = chbook1.Issue(addr1, new(big.Int).SetInt64(-1))
t.Logf("correct error: %v", err)
if err == nil {
t.Fatalf("expected incorrect amount error, got none")
}
received, err = chbox.Receive(ch0)
t.Logf("correct error: %v", err)
if err == nil {
t.Fatalf("expected incorrect amount error, got none and value %v", received)
}
}
func TestDeposit(t *testing.T) {
path0 := filepath.Join(os.TempDir(), "chequebook-test-0.json")
backend := newTestBackend()
contr0, _ := deploy(key0, new(big.Int), backend)
chbook, err := NewChequebook(path0, contr0, key0, backend)
if err != nil {
t.Errorf("expected no error, got %v", err)
}
balance := new(big.Int).SetUint64(42)
chbook.Deposit(balance)
backend.Commit()
if chbook.Balance().Cmp(balance) != 0 {
t.Fatalf("expected balance %v, got %v", balance, chbook.Balance())
}
amount := common.Big1
_, err = chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
exp := new(big.Int).SetUint64(41)
if chbook.Balance().Cmp(exp) != 0 {
t.Fatalf("expected balance %v, got %v", exp, chbook.Balance())
}
// autodeposit on each issue
chbook.AutoDeposit(0, balance, balance)
_, err = chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
_, err = chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
if chbook.Balance().Cmp(balance) != 0 {
t.Fatalf("expected balance %v, got %v", balance, chbook.Balance())
}
// autodeposit off
chbook.AutoDeposit(0, common.Big0, balance)
_, err = chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
_, err = chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
exp = new(big.Int).SetUint64(40)
if chbook.Balance().Cmp(exp) != 0 {
t.Fatalf("expected balance %v, got %v", exp, chbook.Balance())
}
// autodeposit every 200ms if new cheque issued
interval := 200 * time.Millisecond
chbook.AutoDeposit(interval, common.Big1, balance)
_, err = chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
_, err = chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
exp = new(big.Int).SetUint64(38)
if chbook.Balance().Cmp(exp) != 0 {
t.Fatalf("expected balance %v, got %v", exp, chbook.Balance())
}
time.Sleep(3 * interval)
backend.Commit()
if chbook.Balance().Cmp(balance) != 0 {
t.Fatalf("expected balance %v, got %v", balance, chbook.Balance())
}
exp = new(big.Int).SetUint64(40)
chbook.AutoDeposit(4*interval, exp, balance)
_, err = chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
_, err = chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
time.Sleep(3 * interval)
backend.Commit()
if chbook.Balance().Cmp(exp) != 0 {
t.Fatalf("expected balance %v, got %v", exp, chbook.Balance())
}
_, err = chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
time.Sleep(1 * interval)
backend.Commit()
if chbook.Balance().Cmp(balance) != 0 {
t.Fatalf("expected balance %v, got %v", balance, chbook.Balance())
}
chbook.AutoDeposit(1*interval, common.Big0, balance)
chbook.Stop()
_, err = chbook.Issue(addr1, common.Big1)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
_, err = chbook.Issue(addr1, common.Big2)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
time.Sleep(1 * interval)
backend.Commit()
exp = new(big.Int).SetUint64(39)
if chbook.Balance().Cmp(exp) != 0 {
t.Fatalf("expected balance %v, got %v", exp, chbook.Balance())
}
}
func TestCash(t *testing.T) {
path := filepath.Join(os.TempDir(), "chequebook-test.json")
backend := newTestBackend()
contr0, _ := deploy(key0, common.Big2, backend)
chbook, err := NewChequebook(path, contr0, key0, backend)
if err != nil {
t.Errorf("expected no error, got %v", err)
}
chbook.sent[addr1] = new(big.Int).SetUint64(42)
amount := common.Big1
chbook.balance = new(big.Int).Set(common.Big1)
ch, err := chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
chbox, err := NewInbox(key1, contr0, addr1, &key0.PublicKey, backend)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
// cashing latest cheque
if _, err = chbox.Receive(ch); err != nil {
t.Fatalf("expected no error, got %v", err)
}
if _, err = ch.Cash(chbook.session); err != nil {
t.Fatal("Cash failed:", err)
}
backend.Commit()
chbook.balance = new(big.Int).Set(common.Big3)
ch0, err := chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
ch1, err := chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
interval := 10 * time.Millisecond
// setting autocash with interval of 10ms
chbox.AutoCash(interval, nil)
_, err = chbox.Receive(ch0)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
_, err = chbox.Receive(ch1)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
// after 3x interval time and 2 cheques received, exactly one cashing tx is sent
time.Sleep(4 * interval)
backend.Commit()
// after stopping autocash no more tx are sent
ch2, err := chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
chbox.Stop()
_, err = chbox.Receive(ch2)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
time.Sleep(2 * interval)
backend.Commit()
// autocash below 1
chbook.balance = big.NewInt(2)
chbox.AutoCash(0, common.Big1)
ch3, err := chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
ch4, err := chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
_, err = chbox.Receive(ch3)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
_, err = chbox.Receive(ch4)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
// autochash on receipt when maxUncashed is 0
chbook.balance = new(big.Int).Set(common.Big2)
chbox.AutoCash(0, common.Big0)
ch5, err := chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
ch6, err := chbook.Issue(addr1, amount)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
_, err = chbox.Receive(ch5)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
_, err = chbox.Receive(ch6)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
backend.Commit()
}

367
contracts/chequebook/contract/chequebook.go
View file

@ -1,367 +0,0 @@
// Code generated - DO NOT EDIT.
// This file is a generated binding and any manual changes will be lost.
package contract
import (
"math/big"
"strings"
ethereum "github.com/ubiq/go-ubiq"
"github.com/ubiq/go-ubiq/accounts/abi"
"github.com/ubiq/go-ubiq/accounts/abi/bind"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/core/types"
"github.com/ubiq/go-ubiq/event"
)
// ChequebookABI is the input ABI used to generate the binding from.
const ChequebookABI = "[{\"constant\":false,\"inputs\":[],\"name\":\"kill\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[{\"name\":\"\",\"type\":\"address\"}],\"name\":\"sent\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"beneficiary\",\"type\":\"address\"},{\"name\":\"amount\",\"type\":\"uint256\"},{\"name\":\"sig_v\",\"type\":\"uint8\"},{\"name\":\"sig_r\",\"type\":\"bytes32\"},{\"name\":\"sig_s\",\"type\":\"bytes32\"}],\"name\":\"cash\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"payable\":true,\"stateMutability\":\"payable\",\"type\":\"fallback\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"name\":\"deadbeat\",\"type\":\"address\"}],\"name\":\"Overdraft\",\"type\":\"event\"}]"
// ChequebookBin is the compiled bytecode used for deploying new contracts.
const ChequebookBin = `0x606060405260008054600160a060020a033316600160a060020a03199091161790556102ec806100306000396000f3006060604052600436106100565763ffffffff7c010000000000000000000000000000000000000000000000000000000060003504166341c0e1b581146100585780637bf786f81461006b578063fbf788d61461009c575b005b341561006357600080fd5b6100566100ca565b341561007657600080fd5b61008a600160a060020a03600435166100f1565b60405190815260200160405180910390f35b34156100a757600080fd5b610056600160a060020a036004351660243560ff60443516606435608435610103565b60005433600160a060020a03908116911614156100ef57600054600160a060020a0316ff5b565b60016020526000908152604090205481565b600160a060020a0385166000908152600160205260408120548190861161012957600080fd5b3087876040516c01000000000000000000000000600160a060020a03948516810282529290931690910260148301526028820152604801604051809103902091506001828686866040516000815260200160405260006040516020015260405193845260ff90921660208085019190915260408085019290925260608401929092526080909201915160208103908084039060008661646e5a03f115156101cf57600080fd5b505060206040510351600054600160a060020a039081169116146101f257600080fd5b50600160a060020a03808716600090815260016020526040902054860390301631811161026257600160a060020a0387166000818152600160205260409081902088905582156108fc0290839051600060405180830381858888f19350505050151561025d57600080fd5b6102b7565b6000547f2250e2993c15843b32621c89447cc589ee7a9f049c026986e545d3c2c0c6f97890600160a060020a0316604051600160a060020a03909116815260200160405180910390a186600160a060020a0316ff5b505050505050505600a165627a7a72305820533e856fc37e3d64d1706bcc7dfb6b1d490c8d566ea498d9d01ec08965a896ca0029`
// DeployChequebook deploys a new Ubiq contract, binding an instance of Chequebook to it.
func DeployChequebook(auth *bind.TransactOpts, backend bind.ContractBackend) (common.Address, *types.Transaction, *Chequebook, error) {
parsed, err := abi.JSON(strings.NewReader(ChequebookABI))
if err != nil {
return common.Address{}, nil, nil, err
}
address, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(ChequebookBin), backend)
if err != nil {
return common.Address{}, nil, nil, err
}
return address, tx, &Chequebook{ChequebookCaller: ChequebookCaller{contract: contract}, ChequebookTransactor: ChequebookTransactor{contract: contract}, ChequebookFilterer: ChequebookFilterer{contract: contract}}, nil
}
// Chequebook is an auto generated Go binding around an Ubiq contract.
type Chequebook struct {
ChequebookCaller // Read-only binding to the contract
ChequebookTransactor // Write-only binding to the contract
ChequebookFilterer // Log filterer for contract events
}
// ChequebookCaller is an auto generated read-only Go binding around an Ubiq contract.
type ChequebookCaller struct {
contract *bind.BoundContract // Generic contract wrapper for the low level calls
}
// ChequebookTransactor is an auto generated write-only Go binding around an Ubiq contract.
type ChequebookTransactor struct {
contract *bind.BoundContract // Generic contract wrapper for the low level calls
}
// ChequebookFilterer is an auto generated log filtering Go binding around an Ubiq contract events.
type ChequebookFilterer struct {
contract *bind.BoundContract // Generic contract wrapper for the low level calls
}
// ChequebookSession is an auto generated Go binding around an Ubiq contract,
// with pre-set call and transact options.
type ChequebookSession struct {
Contract *Chequebook // Generic contract binding to set the session for
CallOpts bind.CallOpts // Call options to use throughout this session
TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session
}
// ChequebookCallerSession is an auto generated read-only Go binding around an Ubiq contract,
// with pre-set call options.
type ChequebookCallerSession struct {
Contract *ChequebookCaller // Generic contract caller binding to set the session for
CallOpts bind.CallOpts // Call options to use throughout this session
}
// ChequebookTransactorSession is an auto generated write-only Go binding around an Ubiq contract,
// with pre-set transact options.
type ChequebookTransactorSession struct {
Contract *ChequebookTransactor // Generic contract transactor binding to set the session for
TransactOpts bind.TransactOpts // Transaction auth options to use throughout this session
}
// ChequebookRaw is an auto generated low-level Go binding around an Ubiq contract.
type ChequebookRaw struct {
Contract *Chequebook // Generic contract binding to access the raw methods on
}
// ChequebookCallerRaw is an auto generated low-level read-only Go binding around an Ubiq contract.
type ChequebookCallerRaw struct {
Contract *ChequebookCaller // Generic read-only contract binding to access the raw methods on
}
// ChequebookTransactorRaw is an auto generated low-level write-only Go binding around an Ubiq contract.
type ChequebookTransactorRaw struct {
Contract *ChequebookTransactor // Generic write-only contract binding to access the raw methods on
}
// NewChequebook creates a new instance of Chequebook, bound to a specific deployed contract.
func NewChequebook(address common.Address, backend bind.ContractBackend) (*Chequebook, error) {
contract, err := bindChequebook(address, backend, backend, backend)
if err != nil {
return nil, err
}
return &Chequebook{ChequebookCaller: ChequebookCaller{contract: contract}, ChequebookTransactor: ChequebookTransactor{contract: contract}, ChequebookFilterer: ChequebookFilterer{contract: contract}}, nil
}
// NewChequebookCaller creates a new read-only instance of Chequebook, bound to a specific deployed contract.
func NewChequebookCaller(address common.Address, caller bind.ContractCaller) (*ChequebookCaller, error) {
contract, err := bindChequebook(address, caller, nil, nil)
if err != nil {
return nil, err
}
return &ChequebookCaller{contract: contract}, nil
}
// NewChequebookTransactor creates a new write-only instance of Chequebook, bound to a specific deployed contract.
func NewChequebookTransactor(address common.Address, transactor bind.ContractTransactor) (*ChequebookTransactor, error) {
contract, err := bindChequebook(address, nil, transactor, nil)
if err != nil {
return nil, err
}
return &ChequebookTransactor{contract: contract}, nil
}
// NewChequebookFilterer creates a new log filterer instance of Chequebook, bound to a specific deployed contract.
func NewChequebookFilterer(address common.Address, filterer bind.ContractFilterer) (*ChequebookFilterer, error) {
contract, err := bindChequebook(address, nil, nil, filterer)
if err != nil {
return nil, err
}
return &ChequebookFilterer{contract: contract}, nil
}
// bindChequebook binds a generic wrapper to an already deployed contract.
func bindChequebook(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) {
parsed, err := abi.JSON(strings.NewReader(ChequebookABI))
if err != nil {
return nil, err
}
return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil
}
// Call invokes the (constant) contract method with params as input values and
// sets the output to result. The result type might be a single field for simple
// returns, a slice of interfaces for anonymous returns and a struct for named
// returns.
func (_Chequebook *ChequebookRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error {
return _Chequebook.Contract.ChequebookCaller.contract.Call(opts, result, method, params...)
}
// Transfer initiates a plain transaction to move funds to the contract, calling
// its default method if one is available.
func (_Chequebook *ChequebookRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) {
return _Chequebook.Contract.ChequebookTransactor.contract.Transfer(opts)
}
// Transact invokes the (paid) contract method with params as input values.
func (_Chequebook *ChequebookRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) {
return _Chequebook.Contract.ChequebookTransactor.contract.Transact(opts, method, params...)
}
// Call invokes the (constant) contract method with params as input values and
// sets the output to result. The result type might be a single field for simple
// returns, a slice of interfaces for anonymous returns and a struct for named
// returns.
func (_Chequebook *ChequebookCallerRaw) Call(opts *bind.CallOpts, result interface{}, method string, params ...interface{}) error {
return _Chequebook.Contract.contract.Call(opts, result, method, params...)
}
// Transfer initiates a plain transaction to move funds to the contract, calling
// its default method if one is available.
func (_Chequebook *ChequebookTransactorRaw) Transfer(opts *bind.TransactOpts) (*types.Transaction, error) {
return _Chequebook.Contract.contract.Transfer(opts)
}
// Transact invokes the (paid) contract method with params as input values.
func (_Chequebook *ChequebookTransactorRaw) Transact(opts *bind.TransactOpts, method string, params ...interface{}) (*types.Transaction, error) {
return _Chequebook.Contract.contract.Transact(opts, method, params...)
}
// Sent is a free data retrieval call binding the contract method 0x7bf786f8.
//
// Solidity: function sent( address) constant returns(uint256)
func (_Chequebook *ChequebookCaller) Sent(opts *bind.CallOpts, arg0 common.Address) (*big.Int, error) {
var (
ret0 = new(*big.Int)
)
out := ret0
err := _Chequebook.contract.Call(opts, out, "sent", arg0)
return *ret0, err
}
// Sent is a free data retrieval call binding the contract method 0x7bf786f8.
//
// Solidity: function sent( address) constant returns(uint256)
func (_Chequebook *ChequebookSession) Sent(arg0 common.Address) (*big.Int, error) {
return _Chequebook.Contract.Sent(&_Chequebook.CallOpts, arg0)
}
// Sent is a free data retrieval call binding the contract method 0x7bf786f8.
//
// Solidity: function sent( address) constant returns(uint256)
func (_Chequebook *ChequebookCallerSession) Sent(arg0 common.Address) (*big.Int, error) {
return _Chequebook.Contract.Sent(&_Chequebook.CallOpts, arg0)
}
// Cash is a paid mutator transaction binding the contract method 0xfbf788d6.
//
// Solidity: function cash(beneficiary address, amount uint256, sig_v uint8, sig_r bytes32, sig_s bytes32) returns()
func (_Chequebook *ChequebookTransactor) Cash(opts *bind.TransactOpts, beneficiary common.Address, amount *big.Int, sigV uint8, sigR [32]byte, sigS [32]byte) (*types.Transaction, error) {
return _Chequebook.contract.Transact(opts, "cash", beneficiary, amount, sigV, sigR, sigS)
}
// Cash is a paid mutator transaction binding the contract method 0xfbf788d6.
//
// Solidity: function cash(beneficiary address, amount uint256, sig_v uint8, sig_r bytes32, sig_s bytes32) returns()
func (_Chequebook *ChequebookSession) Cash(beneficiary common.Address, amount *big.Int, sigV uint8, sigR [32]byte, sigS [32]byte) (*types.Transaction, error) {
return _Chequebook.Contract.Cash(&_Chequebook.TransactOpts, beneficiary, amount, sigV, sigR, sigS)
}
// Cash is a paid mutator transaction binding the contract method 0xfbf788d6.
//
// Solidity: function cash(beneficiary address, amount uint256, sig_v uint8, sig_r bytes32, sig_s bytes32) returns()
func (_Chequebook *ChequebookTransactorSession) Cash(beneficiary common.Address, amount *big.Int, sigV uint8, sigR [32]byte, sigS [32]byte) (*types.Transaction, error) {
return _Chequebook.Contract.Cash(&_Chequebook.TransactOpts, beneficiary, amount, sigV, sigR, sigS)
}
// Kill is a paid mutator transaction binding the contract method 0x41c0e1b5.
//
// Solidity: function kill() returns()
func (_Chequebook *ChequebookTransactor) Kill(opts *bind.TransactOpts) (*types.Transaction, error) {
return _Chequebook.contract.Transact(opts, "kill")
}
// Kill is a paid mutator transaction binding the contract method 0x41c0e1b5.
//
// Solidity: function kill() returns()
func (_Chequebook *ChequebookSession) Kill() (*types.Transaction, error) {
return _Chequebook.Contract.Kill(&_Chequebook.TransactOpts)
}
// Kill is a paid mutator transaction binding the contract method 0x41c0e1b5.
//
// Solidity: function kill() returns()
func (_Chequebook *ChequebookTransactorSession) Kill() (*types.Transaction, error) {
return _Chequebook.Contract.Kill(&_Chequebook.TransactOpts)
}
// ChequebookOverdraftIterator is returned from FilterOverdraft and is used to iterate over the raw logs and unpacked data for Overdraft events raised by the Chequebook contract.
type ChequebookOverdraftIterator struct {
Event *ChequebookOverdraft // Event containing the contract specifics and raw log
contract *bind.BoundContract // Generic contract to use for unpacking event data
event string // Event name to use for unpacking event data
logs chan types.Log // Log channel receiving the found contract events
sub ethereum.Subscription // Subscription for errors, completion and termination
done bool // Whether the subscription completed delivering logs
fail error // Occurred error to stop iteration
}
// Next advances the iterator to the subsequent event, returning whether there
// are any more events found. In case of a retrieval or parsing error, false is
// returned and Error() can be queried for the exact failure.
func (it *ChequebookOverdraftIterator) Next() bool {
// If the iterator failed, stop iterating
if it.fail != nil {
return false
}
// If the iterator completed, deliver directly whatever's available
if it.done {
select {
case log := <-it.logs:
it.Event = new(ChequebookOverdraft)
if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
it.fail = err
return false
}
it.Event.Raw = log
return true
default:
return false
}
}
// Iterator still in progress, wait for either a data or an error event
select {
case log := <-it.logs:
it.Event = new(ChequebookOverdraft)
if err := it.contract.UnpackLog(it.Event, it.event, log); err != nil {
it.fail = err
return false
}
it.Event.Raw = log
return true
case err := <-it.sub.Err():
it.done = true
it.fail = err
return it.Next()
}
}
// Error retruned any retrieval or parsing error occurred during filtering.
func (it *ChequebookOverdraftIterator) Error() error {
return it.fail
}
// Close terminates the iteration process, releasing any pending underlying
// resources.
func (it *ChequebookOverdraftIterator) Close() error {
it.sub.Unsubscribe()
return nil
}
// ChequebookOverdraft represents a Overdraft event raised by the Chequebook contract.
type ChequebookOverdraft struct {
Deadbeat common.Address
Raw types.Log // Blockchain specific contextual infos
}
// FilterOverdraft is a free log retrieval operation binding the contract event 0x2250e2993c15843b32621c89447cc589ee7a9f049c026986e545d3c2c0c6f978.
//
// Solidity: event Overdraft(deadbeat address)
func (_Chequebook *ChequebookFilterer) FilterOverdraft(opts *bind.FilterOpts) (*ChequebookOverdraftIterator, error) {
logs, sub, err := _Chequebook.contract.FilterLogs(opts, "Overdraft")
if err != nil {
return nil, err
}
return &ChequebookOverdraftIterator{contract: _Chequebook.contract, event: "Overdraft", logs: logs, sub: sub}, nil
}
// WatchOverdraft is a free log subscription operation binding the contract event 0x2250e2993c15843b32621c89447cc589ee7a9f049c026986e545d3c2c0c6f978.
//
// Solidity: event Overdraft(deadbeat address)
func (_Chequebook *ChequebookFilterer) WatchOverdraft(opts *bind.WatchOpts, sink chan<- *ChequebookOverdraft) (event.Subscription, error) {
logs, sub, err := _Chequebook.contract.WatchLogs(opts, "Overdraft")
if err != nil {
return nil, err
}
return event.NewSubscription(func(quit <-chan struct{}) error {
defer sub.Unsubscribe()
for {
select {
case log := <-logs:
// New log arrived, parse the event and forward to the user
event := new(ChequebookOverdraft)
if err := _Chequebook.contract.UnpackLog(event, "Overdraft", log); err != nil {
return err
}
event.Raw = log
select {
case sink <- event:
case err := <-sub.Err():
return err
case <-quit:
return nil
}
case err := <-sub.Err():
return err
case <-quit:
return nil
}
}
}), nil
}

47
contracts/chequebook/contract/chequebook.sol
View file

@ -1,47 +0,0 @@
pragma solidity ^0.4.18;
import "./mortal.sol";
/// @title Chequebook for Ethereum micropayments
/// @author Daniel A. Nagy <daniel@ethereum.org>
contract chequebook is mortal {
// Cumulative paid amount in wei to each beneficiary
mapping (address => uint256) public sent;
/// @notice Overdraft event
event Overdraft(address deadbeat);
// Allow sending ether to the chequebook.
function() public payable { }
/// @notice Cash cheque
///
/// @param beneficiary beneficiary address
/// @param amount cumulative amount in wei
/// @param sig_v signature parameter v
/// @param sig_r signature parameter r
/// @param sig_s signature parameter s
/// The digital signature is calculated on the concatenated triplet of contract address, beneficiary address and cumulative amount
function cash(address beneficiary, uint256 amount, uint8 sig_v, bytes32 sig_r, bytes32 sig_s) public {
// Check if the cheque is old.
// Only cheques that are more recent than the last cashed one are considered.
require(amount > sent[beneficiary]);
// Check the digital signature of the cheque.
bytes32 hash = keccak256(address(this), beneficiary, amount);
require(owner == ecrecover(hash, sig_v, sig_r, sig_s));
// Attempt sending the difference between the cumulative amount on the cheque
// and the cumulative amount on the last cashed cheque to beneficiary.
uint256 diff = amount - sent[beneficiary];
if (diff <= this.balance) {
// update the cumulative amount before sending
sent[beneficiary] = amount;
beneficiary.transfer(diff);
} else {
// Upon failure, punish owner for writing a bounced cheque.
// owner.sendToDebtorsPrison();
Overdraft(owner);
// Compensate beneficiary.
selfdestruct(beneficiary);
}
}
}

5
contracts/chequebook/contract/code.go
View file

@ -1,5 +0,0 @@
package contract
// ContractDeployedCode is used to detect suicides. This constant needs to be
// updated when the contract code is changed.
const ContractDeployedCode = "0x6060604052600436106100565763ffffffff7c010000000000000000000000000000000000000000000000000000000060003504166341c0e1b581146100585780637bf786f81461006b578063fbf788d61461009c575b005b341561006357600080fd5b6100566100ca565b341561007657600080fd5b61008a600160a060020a03600435166100f1565b60405190815260200160405180910390f35b34156100a757600080fd5b610056600160a060020a036004351660243560ff60443516606435608435610103565b60005433600160a060020a03908116911614156100ef57600054600160a060020a0316ff5b565b60016020526000908152604090205481565b600160a060020a0385166000908152600160205260408120548190861161012957600080fd5b3087876040516c01000000000000000000000000600160a060020a03948516810282529290931690910260148301526028820152604801604051809103902091506001828686866040516000815260200160405260006040516020015260405193845260ff90921660208085019190915260408085019290925260608401929092526080909201915160208103908084039060008661646e5a03f115156101cf57600080fd5b505060206040510351600054600160a060020a039081169116146101f257600080fd5b50600160a060020a03808716600090815260016020526040902054860390301631811161026257600160a060020a0387166000818152600160205260409081902088905582156108fc0290839051600060405180830381858888f19350505050151561025d57600080fd5b6102b7565b6000547f2250e2993c15843b32621c89447cc589ee7a9f049c026986e545d3c2c0c6f97890600160a060020a0316604051600160a060020a03909116815260200160405180910390a186600160a060020a0316ff5b505050505050505600a165627a7a72305820533e856fc37e3d64d1706bcc7dfb6b1d490c8d566ea498d9d01ec08965a896ca0029"

10
contracts/chequebook/contract/mortal.sol
View file

@ -1,10 +0,0 @@
pragma solidity ^0.4.0;
import "./owned.sol";
contract mortal is owned {
function kill() public {
if (msg.sender == owner)
selfdestruct(owner);
}
}

15
contracts/chequebook/contract/owned.sol
View file

@ -1,15 +0,0 @@
pragma solidity ^0.4.0;
contract owned {
address owner;
modifier onlyowner() {
if (msg.sender == owner) {
_;
}
}
function owned() public {
owner = msg.sender;
}
}

70
contracts/chequebook/gencode.go
View file

@ -1,70 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// +build none
// This program generates contract/code.go, which contains the chequebook code
// after deployment.
package main
import (
"fmt"
"io/ioutil"
"math/big"
"github.com/ubiq/go-ubiq/accounts/abi/bind"
"github.com/ubiq/go-ubiq/accounts/abi/bind/backends"
"github.com/ubiq/go-ubiq/contracts/chequebook/contract"
"github.com/ubiq/go-ubiq/core"
"github.com/ubiq/go-ubiq/crypto"
)
var (
testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
testAlloc = core.GenesisAlloc{
crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(500000000000)},
}
)
func main() {
backend := backends.NewSimulatedBackend(testAlloc, uint64(100000000))
auth := bind.NewKeyedTransactor(testKey)
// Deploy the contract, get the code.
addr, _, _, err := contract.DeployChequebook(auth, backend)
if err != nil {
panic(err)
}
backend.Commit()
code, err := backend.CodeAt(nil, addr, nil)
if err != nil {
panic(err)
}
if len(code) == 0 {
panic("empty code")
}
// Write the output file.
content := fmt.Sprintf(`package contract
// ContractDeployedCode is used to detect suicides. This constant needs to be
// updated when the contract code is changed.
const ContractDeployedCode = "%#x"
`, code)
if err := ioutil.WriteFile("contract/code.go", []byte(content), 0644); err != nil {
panic(err)
}
}

57
internal/web3ext/web3ext.go
View file

@ -20,7 +20,6 @@ package web3ext
var Modules = map[string]string{
"accounting": Accounting_JS,
"admin": Admin_JS,
"chequebook": Chequebook_JS,
"clique": Clique_JS,
"ubqhash": Ubqhash_JS,
"debug": Debug_JS,
@ -30,41 +29,9 @@ var Modules = map[string]string{
"personal": Personal_JS,
"rpc": RPC_JS,
"shh": Shh_JS,
"swarmfs": SWARMFS_JS,
"txpool": TxPool_JS,
}
const Chequebook_JS = `
web3._extend({
property: 'chequebook',
methods: [
new web3._extend.Method({
name: 'deposit',
call: 'chequebook_deposit',
params: 1,
inputFormatter: [null]
}),
new web3._extend.Property({
name: 'balance',
getter: 'chequebook_balance',
outputFormatter: web3._extend.utils.toDecimal
}),
new web3._extend.Method({
name: 'cash',
call: 'chequebook_cash',
params: 1,
inputFormatter: [null]
}),
new web3._extend.Method({
name: 'issue',
call: 'chequebook_issue',
params: 2,
inputFormatter: [null, null]
}),
]
});
`
const Clique_JS = `
web3._extend({
property: 'clique',
@ -661,30 +628,6 @@ web3._extend({
});
`
const SWARMFS_JS = `
web3._extend({
property: 'swarmfs',
methods:
[
new web3._extend.Method({
name: 'mount',
call: 'swarmfs_mount',
params: 2
}),
new web3._extend.Method({
name: 'unmount',
call: 'swarmfs_unmount',
params: 1
}),
new web3._extend.Method({
name: 'listmounts',
call: 'swarmfs_listmounts',
params: 0
}),
]
});
`
const TxPool_JS = `
web3._extend({
property: 'txpool',

6
p2p/protocols/protocol.go
View file

@ -38,13 +38,13 @@ import (
"sync"
"time"
opentracing "github.com/opentracing/opentracing-go"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/p2p"
"github.com/ubiq/go-ubiq/p2p/spancontext"
"github.com/ubiq/go-ubiq/p2p/tracing"
"github.com/ubiq/go-ubiq/rlp"
"github.com/ubiq/go-ubiq/swarm/spancontext"
"github.com/ubiq/go-ubiq/swarm/tracing"
opentracing "github.com/opentracing/opentracing-go"
)
// error codes used by this protocol scheme

0
swarm/spancontext/spancontext.go → p2p/spancontext/spancontext.go
View file

2
swarm/tracing/tracing.go → p2p/tracing/tracing.go
View file

@ -6,9 +6,9 @@ import (
"strings"
"time"
"github.com/ubiq/go-ubiq/log"
jaeger "github.com/uber/jaeger-client-go"
jaegercfg "github.com/uber/jaeger-client-go/config"
"github.com/ubiq/go-ubiq/log"
cli "gopkg.in/urfave/cli.v1"
)

35
swarm/AUTHORS
View file

@ -1,35 +0,0 @@
# Core team members
Viktor Trón - @zelig
Louis Holbrook - @nolash
Lewis Marshall - @lmars
Anton Evangelatov - @nonsense
Janoš Guljaš - @janos
Balint Gabor - @gbalint
Elad Nachmias - @justelad
Daniel A. Nagy - @nagydani
Aron Fischer - @homotopycolimit
Fabio Barone - @holisticode
Zahoor Mohamed - @jmozah
Zsolt Felföldi - @zsfelfoldi
# External contributors
Kiel Barry
Gary Rong
Jared Wasinger
Leon Stanko
Javier Peletier [epiclabs.io]
Bartek Borkowski [tungsten-labs.com]
Shane Howley [mainframe.com]
Doug Leonard [mainframe.com]
Ivan Daniluk [status.im]
Felix Lange [EF]
Martin Holst Swende [EF]
Guillaume Ballet [EF]
ligi [EF]
Christopher Dro [blick-labs.com]
Sergii Bomko [ledgerleopard.com]
Domino Valdano
Rafael Matias
Coogan Brennan

25
swarm/OWNERS
View file

@ -1,25 +0,0 @@
# Ownership by go packages
swarm
├── api ─────────────────── ethersphere
├── bmt ─────────────────── @zelig
├── dev ─────────────────── @lmars
├── fuse ────────────────── @jmozah, @holisticode
├── grafana_dashboards ──── @nonsense
├── metrics ─────────────── @nonsense, @holisticode
├── network ─────────────── ethersphere
│ ├── bitvector ───────── @zelig, @janos, @gbalint
│ ├── priorityqueue ───── @zelig, @janos, @gbalint
│ ├── simulations ─────── @zelig
│ └── stream ──────────── @janos, @zelig, @gbalint, @holisticode, @justelad
│ ├── intervals ───── @janos
│ └── testing ─────── @zelig
├── pot ─────────────────── @zelig
├── pss ─────────────────── @nolash, @zelig, @nonsense
├── services ────────────── @zelig
├── state ───────────────── @justelad
├── storage ─────────────── ethersphere
│ ├── encryption ──────── @gbalint, @zelig, @nagydani
│ ├── mock ────────────── @janos
│ └── feed ────────────── @nolash, @jpeletier
└── testutil ────────────── @lmars

244
swarm/README.md
View file

@ -1,244 +0,0 @@
## Swarm
[https://swarm.ubiq.org](https://swarm.ubiq.org)
Swarm is a distributed storage platform and content distribution service, a native base layer service of the ethereum web3 stack. The primary objective of Swarm is to provide a decentralized and redundant store for dapp code and data as well as block chain and state data. Swarm is also set out to provide various base layer services for web3, including node-to-node messaging, media streaming, decentralised database services and scalable state-channel infrastructure for decentralised service economies.
[![Travis](https://travis-ci.org/ubiq/go-ubiq.svg?branch=master)](https://travis-ci.org/ubiq/go-ubiq)
[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethersphere/orange-lounge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
## Table of Contents
* [Building the source](#building-the-source)
* [Running Swarm](#running-swarm)
* [Documentation](#documentation)
* [Developers Guide](#developers-guide)
* [Go Environment](#development-environment)
* [Vendored Dependencies](#vendored-dependencies)
* [Testing](#testing)
* [Profiling Swarm](#profiling-swarm)
* [Metrics and Instrumentation in Swarm](#metrics-and-instrumentation-in-swarm)
* [Public Gateways](#public-gateways)
* [Swarm Dapps](#swarm-dapps)
* [Contributing](#contributing)
* [License](#license)
## Building the source
Building Swarm requires Go (version 1.10 or later).
go get -d github.com/ubiq/go-ubiq
go install github.com/ubiq/go-ubiq/cmd/swarm
## Running Swarm
Going through all the possible command line flags is out of scope here, but we've enumerated a few common parameter combos to get you up to speed quickly on how you can run your own Swarm node.
To run Swarm you need an Ethereum account. You can create a new account by running the following command:
gubiq account new
You will be prompted for a password:
Your new account is locked with a password. Please give a password. Do not forget this password.
Passphrase:
Repeat passphrase:
Once you have specified the password, the output will be the Ethereum address representing that account. For example:
Address: {2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1}
Using this account, connect to Swarm with
swarm --bzzaccount <your-account-here>
# in our example
swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1
### Verifying that your local Swarm node is running
When running, Swarm is accessible through an HTTP API on port 8500.
Confirm that it is up and running by pointing your browser to http://localhost:8500
### Ethereum Name Service resolution
The Ethereum Name Service is the Ethereum equivalent of DNS in the classic web. In order to use ENS to resolve names to Swarm content hashes (e.g. `bzz://theswarm.eth`), `swarm` has to connect to a `gubiq` instance, which is synced with the Ethereum mainnet. This is done using the `--ens-api` flag.
swarm --bzzaccount <your-account-here> \
--ens-api '$HOME/.ubiq/gubiq.ipc'
# in our example
swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1 \
--ens-api '$HOME/.ubiq/gubiq.ipc'
For more information on usage, features or command line flags, please consult the Documentation.
## Documentation
Swarm documentation can be found at [https://swarm-guide.readthedocs.io](https://swarm-guide.readthedocs.io).
## Developers Guide
### Go Environment
We assume that you have Go v1.10 installed, and `GOPATH` is set.
You must have your working copy under `$GOPATH/src/github.com/ubiq/go-ubiq`.
Most likely you will be working from your fork of `go-ethereum`, let's say from `github.com/nirname/go-ethereum`. Clone or move your fork into the right place:
```
git clone git@github.com:nirname/go-ethereum.git $GOPATH/src/github.com/ubiq/go-ubiq
```
### Vendored Dependencies
All dependencies are tracked in the `vendor` directory. We use `govendor` to manage them.
If you want to add a new dependency, run `govendor fetch <import-path>`, then commit the result.
If you want to update all dependencies to their latest upstream version, run `govendor fetch +v`.
### Testing
This section explains how to run unit, integration, and end-to-end tests in your development sandbox.
Testing one library:
```
go test -v -cpu 4 ./swarm/api
```
Note: Using options -cpu (number of cores allowed) and -v (logging even if no error) is recommended.
Testing only some methods:
```
go test -v -cpu 4 ./eth -run TestMethod
```
Note: here all tests with prefix TestMethod will be run, so if you got TestMethod, TestMethod1, then both!
Running benchmarks:
```
go test -v -cpu 4 -bench . -run BenchmarkJoin
```
### Profiling Swarm
This section explains how to add Go `pprof` profiler to Swarm
If `swarm` is started with the `--pprof` option, a debugging HTTP server is made available on port 6060.
You can bring up http://localhost:6060/debug/pprof to see the heap, running routines etc.
By clicking full goroutine stack dump (clicking http://localhost:6060/debug/pprof/goroutine?debug=2) you can generate trace that is useful for debugging.
### Metrics and Instrumentation in Swarm
This section explains how to visualize and use existing Swarm metrics and how to instrument Swarm with a new metric.
Swarm metrics system is based on the `go-metrics` library.
The most common types of measurements we use in Swarm are `counters` and `resetting timers`. Consult the `go-metrics` documentation for full reference of available types.
```
# incrementing a counter
metrics.GetOrRegisterCounter("network.stream.received_chunks", nil).Inc(1)
# measuring latency with a resetting timer
start := time.Now()
t := metrics.GetOrRegisterResettingTimer("http.request.GET.time"), nil)
...
t := UpdateSince(start)
```
#### Visualizing metrics
Swarm supports an InfluxDB exporter. Consult the help section to learn about the command line arguments used to configure it:
```
swarm --help | grep metrics
```
We use Grafana and InfluxDB to visualise metrics reported by Swarm. We keep our Grafana dashboards under version control at `./swarm/grafana_dashboards`. You could use them or design your own.
We have built a tool to help with automatic start of Grafana and InfluxDB and provisioning of dashboards at https://github.com/nonsense/stateth , which requires that you have Docker installed.
Once you have `stateth` installed, and you have Docker running locally, you have to:
1. Run `stateth` and keep it running in the background
```
stateth --rm --grafana-dashboards-folder $GOPATH/src/github.com/ubiq/go-ubiq/swarm/grafana_dashboards --influxdb-database metrics
```
2. Run `swarm` with at least the following params:
```
--metrics \
--metrics.influxdb.export \
--metrics.influxdb.endpoint "http://localhost:8086" \
--metrics.influxdb.username "admin" \
--metrics.influxdb.password "admin" \
--metrics.influxdb.database "metrics"
```
3. Open Grafana at http://localhost:3000 and view the dashboards to gain insight into Swarm.
## Public Gateways
Swarm offers a local HTTP proxy API that Dapps can use to interact with Swarm. The Ethereum Foundation is hosting a public gateway, which allows free access so that people can try Swarm without running their own node.
The Swarm public gateways are temporary and users should not rely on their existence for production services.
The Swarm public gateway can be found at https://swarm-gateways.net and is always running the latest `stable` Swarm release.
## Swarm Dapps
You can find a few reference Swarm decentralised applications at: https://swarm-gateways.net/bzz:/swarmapps.eth
Their source code can be found at: https://github.com/ethersphere/swarm-dapps
## Contributing
Thank you for considering to help out with the source code! We welcome contributions from
anyone on the internet, and are grateful for even the smallest of fixes!
If you'd like to contribute to Swarm, please fork, fix, commit and send a pull request
for the maintainers to review and merge into the main code base. If you wish to submit more
complex changes though, please check up with the core devs first on [our Swarm gitter channel](https://gitter.im/ethersphere/orange-lounge)
to ensure those changes are in line with the general philosophy of the project and/or get some
early feedback which can make both your efforts much lighter as well as our review and merge
procedures quick and simple.
Please make sure your contributions adhere to our coding guidelines:
* Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)).
* Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) guidelines.
* Pull requests need to be based on and opened against the `master` branch.
* [Code review guidelines](https://github.com/ubiq/go-ubiq/wiki/Code-Review-Guidelines).
* Commit messages should be prefixed with the package(s) they modify.
* E.g. "swarm/fuse: ignore default manifest entry"
## License
The go-ethereum library (i.e. all code outside of the `cmd` directory) is licensed under the
[GNU Lesser General Public License v3.0](https://www.gnu.org/licenses/lgpl-3.0.en.html), also
included in our repository in the `COPYING.LESSER` file.
The go-ethereum binaries (i.e. all code inside of the `cmd` directory) is licensed under the
[GNU General Public License v3.0](https://www.gnu.org/licenses/gpl-3.0.en.html), also included
in our repository in the `COPYING` file.

538
swarm/api/act.go
View file

@ -1,538 +0,0 @@
package api
import (
"context"
"crypto/ecdsa"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/crypto"
"github.com/ubiq/go-ubiq/crypto/ecies"
"github.com/ubiq/go-ubiq/swarm/log"
"github.com/ubiq/go-ubiq/swarm/sctx"
"github.com/ubiq/go-ubiq/swarm/storage"
"golang.org/x/crypto/scrypt"
"golang.org/x/crypto/sha3"
cli "gopkg.in/urfave/cli.v1"
)
var (
ErrDecrypt = errors.New("cant decrypt - forbidden")
ErrUnknownAccessType = errors.New("unknown access type (or not implemented)")
ErrDecryptDomainForbidden = errors.New("decryption request domain forbidden - can only decrypt on localhost")
AllowedDecryptDomains = []string{
"localhost",
"127.0.0.1",
}
)
const EMPTY_CREDENTIALS = ""
type AccessEntry struct {
Type AccessType
Publisher string
Salt []byte
Act string
KdfParams *KdfParams
}
type DecryptFunc func(*ManifestEntry) error
func (a *AccessEntry) MarshalJSON() (out []byte, err error) {
return json.Marshal(struct {
Type AccessType `json:"type,omitempty"`
Publisher string `json:"publisher,omitempty"`
Salt string `json:"salt,omitempty"`
Act string `json:"act,omitempty"`
KdfParams *KdfParams `json:"kdf_params,omitempty"`
}{
Type: a.Type,
Publisher: a.Publisher,
Salt: hex.EncodeToString(a.Salt),
Act: a.Act,
KdfParams: a.KdfParams,
})
}
func (a *AccessEntry) UnmarshalJSON(value []byte) error {
v := struct {
Type AccessType `json:"type,omitempty"`
Publisher string `json:"publisher,omitempty"`
Salt string `json:"salt,omitempty"`
Act string `json:"act,omitempty"`
KdfParams *KdfParams `json:"kdf_params,omitempty"`
}{}
err := json.Unmarshal(value, &v)
if err != nil {
return err
}
a.Act = v.Act
a.KdfParams = v.KdfParams
a.Publisher = v.Publisher
a.Salt, err = hex.DecodeString(v.Salt)
if err != nil {
return err
}
if len(a.Salt) != 32 {
return errors.New("salt should be 32 bytes long")
}
a.Type = v.Type
return nil
}
type KdfParams struct {
N int `json:"n"`
P int `json:"p"`
R int `json:"r"`
}
type AccessType string
const AccessTypePass = AccessType("pass")
const AccessTypePK = AccessType("pk")
const AccessTypeACT = AccessType("act")
// NewAccessEntryPassword creates a manifest AccessEntry in order to create an ACT protected by a password
func NewAccessEntryPassword(salt []byte, kdfParams *KdfParams) (*AccessEntry, error) {
if len(salt) != 32 {
return nil, fmt.Errorf("salt should be 32 bytes long")
}
return &AccessEntry{
Type: AccessTypePass,
Salt: salt,
KdfParams: kdfParams,
}, nil
}
// NewAccessEntryPK creates a manifest AccessEntry in order to create an ACT protected by a pair of Elliptic Curve keys
func NewAccessEntryPK(publisher string, salt []byte) (*AccessEntry, error) {
if len(publisher) != 66 {
return nil, fmt.Errorf("publisher should be 66 characters long, got %d", len(publisher))
}
if len(salt) != 32 {
return nil, fmt.Errorf("salt should be 32 bytes long")
}
return &AccessEntry{
Type: AccessTypePK,
Publisher: publisher,
Salt: salt,
}, nil
}
// NewAccessEntryACT creates a manifest AccessEntry in order to create an ACT protected by a combination of EC keys and passwords
func NewAccessEntryACT(publisher string, salt []byte, act string) (*AccessEntry, error) {
if len(salt) != 32 {
return nil, fmt.Errorf("salt should be 32 bytes long")
}
if len(publisher) != 66 {
return nil, fmt.Errorf("publisher should be 66 characters long")
}
return &AccessEntry{
Type: AccessTypeACT,
Publisher: publisher,
Salt: salt,
Act: act,
KdfParams: DefaultKdfParams,
}, nil
}
// NOOPDecrypt is a generic decrypt function that is passed into the API in places where real ACT decryption capabilities are
// either unwanted, or alternatively, cannot be implemented in the immediate scope
func NOOPDecrypt(*ManifestEntry) error {
return nil
}
var DefaultKdfParams = NewKdfParams(262144, 1, 8)
// NewKdfParams returns a KdfParams struct with the given scrypt params
func NewKdfParams(n, p, r int) *KdfParams {
return &KdfParams{
N: n,
P: p,
R: r,
}
}
// NewSessionKeyPassword creates a session key based on a shared secret (password) and the given salt
// and kdf parameters in the access entry
func NewSessionKeyPassword(password string, accessEntry *AccessEntry) ([]byte, error) {
if accessEntry.Type != AccessTypePass && accessEntry.Type != AccessTypeACT {
return nil, errors.New("incorrect access entry type")
}
return sessionKeyPassword(password, accessEntry.Salt, accessEntry.KdfParams)
}
func sessionKeyPassword(password string, salt []byte, kdfParams *KdfParams) ([]byte, error) {
return scrypt.Key(
[]byte(password),
salt,
kdfParams.N,
kdfParams.R,
kdfParams.P,
32,
)
}
// NewSessionKeyPK creates a new ACT Session Key using an ECDH shared secret for the given key pair and the given salt value
func NewSessionKeyPK(private *ecdsa.PrivateKey, public *ecdsa.PublicKey, salt []byte) ([]byte, error) {
granteePubEcies := ecies.ImportECDSAPublic(public)
privateKey := ecies.ImportECDSA(private)
bytes, err := privateKey.GenerateShared(granteePubEcies, 16, 16)
if err != nil {
return nil, err
}
bytes = append(salt, bytes...)
sessionKey := crypto.Keccak256(bytes)
return sessionKey, nil
}
func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.PrivateKey) DecryptFunc {
return func(m *ManifestEntry) error {
if m.Access == nil {
return nil
}
allowed := false
requestDomain := sctx.GetHost(ctx)
for _, v := range AllowedDecryptDomains {
if strings.Contains(requestDomain, v) {
allowed = true
}
}
if !allowed {
return ErrDecryptDomainForbidden
}
switch m.Access.Type {
case "pass":
if credentials != "" {
key, err := NewSessionKeyPassword(credentials, m.Access)
if err != nil {
return err
}
ref, err := hex.DecodeString(m.Hash)
if err != nil {
return err
}
enc := NewRefEncryption(len(ref) - 8)
decodedRef, err := enc.Decrypt(ref, key)
if err != nil {
return ErrDecrypt
}
m.Hash = hex.EncodeToString(decodedRef)
m.Access = nil
return nil
}
return ErrDecrypt
case "pk":
publisherBytes, err := hex.DecodeString(m.Access.Publisher)
if err != nil {
return ErrDecrypt
}
publisher, err := crypto.DecompressPubkey(publisherBytes)
if err != nil {
return ErrDecrypt
}
key, err := NewSessionKeyPK(pk, publisher, m.Access.Salt)
if err != nil {
return ErrDecrypt
}
ref, err := hex.DecodeString(m.Hash)
if err != nil {
return err
}
enc := NewRefEncryption(len(ref) - 8)
decodedRef, err := enc.Decrypt(ref, key)
if err != nil {
return ErrDecrypt
}
m.Hash = hex.EncodeToString(decodedRef)
m.Access = nil
return nil
case "act":
var (
sessionKey []byte
err error
)
publisherBytes, err := hex.DecodeString(m.Access.Publisher)
if err != nil {
return ErrDecrypt
}
publisher, err := crypto.DecompressPubkey(publisherBytes)
if err != nil {
return ErrDecrypt
}
sessionKey, err = NewSessionKeyPK(pk, publisher, m.Access.Salt)
if err != nil {
return ErrDecrypt
}
found, ciphertext, decryptionKey, err := a.getACTDecryptionKey(ctx, storage.Address(common.Hex2Bytes(m.Access.Act)), sessionKey)
if err != nil {
return err
}
if !found {
// try to fall back to password
if credentials != "" {
sessionKey, err = NewSessionKeyPassword(credentials, m.Access)
if err != nil {
return err
}
found, ciphertext, decryptionKey, err = a.getACTDecryptionKey(ctx, storage.Address(common.Hex2Bytes(m.Access.Act)), sessionKey)
if err != nil {
return err
}
if !found {
return ErrDecrypt
}
} else {
return ErrDecrypt
}
}
enc := NewRefEncryption(len(ciphertext) - 8)
decodedRef, err := enc.Decrypt(ciphertext, decryptionKey)
if err != nil {
return ErrDecrypt
}
ref, err := hex.DecodeString(m.Hash)
if err != nil {
return err
}
enc = NewRefEncryption(len(ref) - 8)
decodedMainRef, err := enc.Decrypt(ref, decodedRef)
if err != nil {
return ErrDecrypt
}
m.Hash = hex.EncodeToString(decodedMainRef)
m.Access = nil
return nil
}
return ErrUnknownAccessType
}
}
func (a *API) getACTDecryptionKey(ctx context.Context, actManifestAddress storage.Address, sessionKey []byte) (found bool, ciphertext, decryptionKey []byte, err error) {
hasher := sha3.NewLegacyKeccak256()
hasher.Write(append(sessionKey, 0))
lookupKey := hasher.Sum(nil)
hasher.Reset()
hasher.Write(append(sessionKey, 1))
accessKeyDecryptionKey := hasher.Sum(nil)
hasher.Reset()
lk := hex.EncodeToString(lookupKey)
list, err := a.GetManifestList(ctx, NOOPDecrypt, actManifestAddress, lk)
if err != nil {
return false, nil, nil, err
}
for _, v := range list.Entries {
if v.Path == lk {
cipherTextBytes, err := hex.DecodeString(v.Hash)
if err != nil {
return false, nil, nil, err
}
return true, cipherTextBytes, accessKeyDecryptionKey, nil
}
}
return false, nil, nil, nil
}
func GenerateAccessControlManifest(ctx *cli.Context, ref string, accessKey []byte, ae *AccessEntry) (*Manifest, error) {
refBytes, err := hex.DecodeString(ref)
if err != nil {
return nil, err
}
// encrypt ref with accessKey
enc := NewRefEncryption(len(refBytes))
encrypted, err := enc.Encrypt(refBytes, accessKey)
if err != nil {
return nil, err
}
m := &Manifest{
Entries: []ManifestEntry{
{
Hash: hex.EncodeToString(encrypted),
ContentType: ManifestType,
ModTime: time.Now(),
Access: ae,
},
},
}
return m, nil
}
// DoPK is a helper function to the CLI API that handles the entire business logic for
// creating a session key and access entry given the cli context, ec keys and salt
func DoPK(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) {
if granteePublicKey == "" {
return nil, nil, errors.New("need a grantee Public Key")
}
b, err := hex.DecodeString(granteePublicKey)
if err != nil {
log.Error("error decoding grantee public key", "err", err)
return nil, nil, err
}
granteePub, err := crypto.DecompressPubkey(b)
if err != nil {
log.Error("error decompressing grantee public key", "err", err)
return nil, nil, err
}
sessionKey, err = NewSessionKeyPK(privateKey, granteePub, salt)
if err != nil {
log.Error("error getting session key", "err", err)
return nil, nil, err
}
ae, err = NewAccessEntryPK(hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)), salt)
if err != nil {
log.Error("error generating access entry", "err", err)
return nil, nil, err
}
return sessionKey, ae, nil
}
// DoACT is a helper function to the CLI API that handles the entire business logic for
// creating a access key, access entry and ACT manifest (including uploading it) given the cli context, ec keys, password grantees and salt
func DoACT(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees []string, encryptPasswords []string) (accessKey []byte, ae *AccessEntry, actManifest *Manifest, err error) {
if len(grantees) == 0 && len(encryptPasswords) == 0 {
return nil, nil, nil, errors.New("did not get any grantee public keys or any encryption passwords")
}
publisherPub := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey))
grantees = append(grantees, publisherPub)
accessKey = make([]byte, 32)
if _, err := io.ReadFull(rand.Reader, salt); err != nil {
panic("reading from crypto/rand failed: " + err.Error())
}
if _, err := io.ReadFull(rand.Reader, accessKey); err != nil {
panic("reading from crypto/rand failed: " + err.Error())
}
lookupPathEncryptedAccessKeyMap := make(map[string]string)
i := 0
for _, v := range grantees {
i++
if v == "" {
return nil, nil, nil, errors.New("need a grantee Public Key")
}
b, err := hex.DecodeString(v)
if err != nil {
log.Error("error decoding grantee public key", "err", err)
return nil, nil, nil, err
}
granteePub, err := crypto.DecompressPubkey(b)
if err != nil {
log.Error("error decompressing grantee public key", "err", err)
return nil, nil, nil, err
}
sessionKey, err := NewSessionKeyPK(privateKey, granteePub, salt)
if err != nil {
return nil, nil, nil, err
}
hasher := sha3.NewLegacyKeccak256()
hasher.Write(append(sessionKey, 0))
lookupKey := hasher.Sum(nil)
hasher.Reset()
hasher.Write(append(sessionKey, 1))
accessKeyEncryptionKey := hasher.Sum(nil)
enc := NewRefEncryption(len(accessKey))
encryptedAccessKey, err := enc.Encrypt(accessKey, accessKeyEncryptionKey)
if err != nil {
return nil, nil, nil, err
}
lookupPathEncryptedAccessKeyMap[hex.EncodeToString(lookupKey)] = hex.EncodeToString(encryptedAccessKey)
}
for _, pass := range encryptPasswords {
sessionKey, err := sessionKeyPassword(pass, salt, DefaultKdfParams)
if err != nil {
return nil, nil, nil, err
}
hasher := sha3.NewLegacyKeccak256()
hasher.Write(append(sessionKey, 0))
lookupKey := hasher.Sum(nil)
hasher.Reset()
hasher.Write(append(sessionKey, 1))
accessKeyEncryptionKey := hasher.Sum(nil)
enc := NewRefEncryption(len(accessKey))
encryptedAccessKey, err := enc.Encrypt(accessKey, accessKeyEncryptionKey)
if err != nil {
return nil, nil, nil, err
}
lookupPathEncryptedAccessKeyMap[hex.EncodeToString(lookupKey)] = hex.EncodeToString(encryptedAccessKey)
}
m := &Manifest{
Entries: []ManifestEntry{},
}
for k, v := range lookupPathEncryptedAccessKeyMap {
m.Entries = append(m.Entries, ManifestEntry{
Path: k,
Hash: v,
ContentType: "text/plain",
})
}
ae, err = NewAccessEntryACT(hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)), salt, "")
if err != nil {
return nil, nil, nil, err
}
return accessKey, ae, m, nil
}
// DoPassword is a helper function to the CLI API that handles the entire business logic for
// creating a session key and an access entry given the cli context, password and salt.
// By default - DefaultKdfParams are used as the scrypt params
func DoPassword(ctx *cli.Context, password string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) {
ae, err = NewAccessEntryPassword(salt, DefaultKdfParams)
if err != nil {
return nil, nil, err
}
sessionKey, err = NewSessionKeyPassword(password, ae)
if err != nil {
return nil, nil, err
}
return sessionKey, ae, nil
}

1017
swarm/api/api.go
View file

File diff suppressed because it is too large Load diff

502
swarm/api/api_test.go
View file

@ -1,502 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"bytes"
"context"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"math/big"
"os"
"testing"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/core/types"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/swarm/sctx"
"github.com/ubiq/go-ubiq/swarm/storage"
)
func init() {
loglevel := flag.Int("loglevel", 2, "loglevel")
flag.Parse()
log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
}
func testAPI(t *testing.T, f func(*API, bool)) {
datadir, err := ioutil.TempDir("", "bzz-test")
if err != nil {
t.Fatalf("unable to create temp dir: %v", err)
}
defer os.RemoveAll(datadir)
fileStore, err := storage.NewLocalFileStore(datadir, make([]byte, 32))
if err != nil {
return
}
api := NewAPI(fileStore, nil, nil, nil)
f(api, false)
f(api, true)
}
type testResponse struct {
reader storage.LazySectionReader
*Response
}
func checkResponse(t *testing.T, resp *testResponse, exp *Response) {
if resp.MimeType != exp.MimeType {
t.Errorf("incorrect mimeType. expected '%s', got '%s'", exp.MimeType, resp.MimeType)
}
if resp.Status != exp.Status {
t.Errorf("incorrect status. expected '%d', got '%d'", exp.Status, resp.Status)
}
if resp.Size != exp.Size {
t.Errorf("incorrect size. expected '%d', got '%d'", exp.Size, resp.Size)
}
if resp.reader != nil {
content := make([]byte, resp.Size)
read, _ := resp.reader.Read(content)
if int64(read) != exp.Size {
t.Errorf("incorrect content length. expected '%d...', got '%d...'", read, exp.Size)
}
resp.Content = string(content)
}
if resp.Content != exp.Content {
// if !bytes.Equal(resp.Content, exp.Content)
t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Content))
}
}
// func expResponse(content []byte, mimeType string, status int) *Response {
func expResponse(content string, mimeType string, status int) *Response {
log.Trace(fmt.Sprintf("expected content (%v): %v ", len(content), content))
return &Response{mimeType, status, int64(len(content)), content}
}
func testGet(t *testing.T, api *API, bzzhash, path string) *testResponse {
addr := storage.Address(common.Hex2Bytes(bzzhash))
reader, mimeType, status, _, err := api.Get(context.TODO(), NOOPDecrypt, addr, path)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
quitC := make(chan bool)
size, err := reader.Size(context.TODO(), quitC)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
log.Trace(fmt.Sprintf("reader size: %v ", size))
s := make([]byte, size)
_, err = reader.Read(s)
if err != io.EOF {
t.Fatalf("unexpected error: %v", err)
}
reader.Seek(0, 0)
return &testResponse{reader, &Response{mimeType, status, size, string(s)}}
// return &testResponse{reader, &Response{mimeType, status, reader.Size(), nil}}
}
func TestApiPut(t *testing.T) {
testAPI(t, func(api *API, toEncrypt bool) {
content := "hello"
exp := expResponse(content, "text/plain", 0)
ctx := context.TODO()
addr, wait, err := api.Put(ctx, content, exp.MimeType, toEncrypt)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
err = wait(ctx)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
resp := testGet(t, api, addr.Hex(), "")
checkResponse(t, resp, exp)
})
}
// testResolver implements the Resolver interface and either returns the given
// hash if it is set, or returns a "name not found" error
type testResolveValidator struct {
hash *common.Hash
}
func newTestResolveValidator(addr string) *testResolveValidator {
r := &testResolveValidator{}
if addr != "" {
hash := common.HexToHash(addr)
r.hash = &hash
}
return r
}
func (t *testResolveValidator) Resolve(addr string) (common.Hash, error) {
if t.hash == nil {
return common.Hash{}, fmt.Errorf("DNS name not found: %q", addr)
}
return *t.hash, nil
}
func (t *testResolveValidator) Owner(node [32]byte) (addr common.Address, err error) {
return
}
func (t *testResolveValidator) HeaderByNumber(context.Context, *big.Int) (header *types.Header, err error) {
return
}
// TestAPIResolve tests resolving URIs which can either contain content hashes
// or ENS names
func TestAPIResolve(t *testing.T) {
ensAddr := "swarm.eth"
hashAddr := "1111111111111111111111111111111111111111111111111111111111111111"
resolvedAddr := "2222222222222222222222222222222222222222222222222222222222222222"
doesResolve := newTestResolveValidator(resolvedAddr)
doesntResolve := newTestResolveValidator("")
type test struct {
desc string
dns Resolver
addr string
immutable bool
result string
expectErr error
}
tests := []*test{
{
desc: "DNS not configured, hash address, returns hash address",
dns: nil,
addr: hashAddr,
result: hashAddr,
},
{
desc: "DNS not configured, ENS address, returns error",
dns: nil,
addr: ensAddr,
expectErr: errors.New(`no DNS to resolve name: "swarm.eth"`),
},
{
desc: "DNS configured, hash address, hash resolves, returns resolved address",
dns: doesResolve,
addr: hashAddr,
result: resolvedAddr,
},
{
desc: "DNS configured, immutable hash address, hash resolves, returns hash address",
dns: doesResolve,
addr: hashAddr,
immutable: true,
result: hashAddr,
},
{
desc: "DNS configured, hash address, hash doesn't resolve, returns hash address",
dns: doesntResolve,
addr: hashAddr,
result: hashAddr,
},
{
desc: "DNS configured, ENS address, name resolves, returns resolved address",
dns: doesResolve,
addr: ensAddr,
result: resolvedAddr,
},
{
desc: "DNS configured, immutable ENS address, name resolves, returns error",
dns: doesResolve,
addr: ensAddr,
immutable: true,
expectErr: errors.New(`immutable address not a content hash: "swarm.eth"`),
},
{
desc: "DNS configured, ENS address, name doesn't resolve, returns error",
dns: doesntResolve,
addr: ensAddr,
expectErr: errors.New(`DNS name not found: "swarm.eth"`),
},
}
for _, x := range tests {
t.Run(x.desc, func(t *testing.T) {
api := &API{dns: x.dns}
uri := &URI{Addr: x.addr, Scheme: "bzz"}
if x.immutable {
uri.Scheme = "bzz-immutable"
}
res, err := api.ResolveURI(context.TODO(), uri, "")
if err == nil {
if x.expectErr != nil {
t.Fatalf("expected error %q, got result %q", x.expectErr, res)
}
if res.String() != x.result {
t.Fatalf("expected result %q, got %q", x.result, res)
}
} else {
if x.expectErr == nil {
t.Fatalf("expected no error, got %q", err)
}
if err.Error() != x.expectErr.Error() {
t.Fatalf("expected error %q, got %q", x.expectErr, err)
}
}
})
}
}
func TestMultiResolver(t *testing.T) {
doesntResolve := newTestResolveValidator("")
ethAddr := "swarm.eth"
ethHash := "0x2222222222222222222222222222222222222222222222222222222222222222"
ethResolve := newTestResolveValidator(ethHash)
testAddr := "swarm.test"
testHash := "0x1111111111111111111111111111111111111111111111111111111111111111"
testResolve := newTestResolveValidator(testHash)
tests := []struct {
desc string
r Resolver
addr string
result string
err error
}{
{
desc: "No resolvers, returns error",
r: NewMultiResolver(),
err: NewNoResolverError(""),
},
{
desc: "One default resolver, returns resolved address",
r: NewMultiResolver(MultiResolverOptionWithResolver(ethResolve, "")),
addr: ethAddr,
result: ethHash,
},
{
desc: "Two default resolvers, returns resolved address",
r: NewMultiResolver(
MultiResolverOptionWithResolver(ethResolve, ""),
MultiResolverOptionWithResolver(ethResolve, ""),
),
addr: ethAddr,
result: ethHash,
},
{
desc: "Two default resolvers, first doesn't resolve, returns resolved address",
r: NewMultiResolver(
MultiResolverOptionWithResolver(doesntResolve, ""),
MultiResolverOptionWithResolver(ethResolve, ""),
),
addr: ethAddr,
result: ethHash,
},
{
desc: "Default resolver doesn't resolve, tld resolver resolve, returns resolved address",
r: NewMultiResolver(
MultiResolverOptionWithResolver(doesntResolve, ""),
MultiResolverOptionWithResolver(ethResolve, "eth"),
),
addr: ethAddr,
result: ethHash,
},
{
desc: "Three TLD resolvers, third resolves, returns resolved address",
r: NewMultiResolver(
MultiResolverOptionWithResolver(doesntResolve, "eth"),
MultiResolverOptionWithResolver(doesntResolve, "eth"),
MultiResolverOptionWithResolver(ethResolve, "eth"),
),
addr: ethAddr,
result: ethHash,
},
{
desc: "One TLD resolver doesn't resolve, returns error",
r: NewMultiResolver(
MultiResolverOptionWithResolver(doesntResolve, ""),
MultiResolverOptionWithResolver(ethResolve, "eth"),
),
addr: ethAddr,
result: ethHash,
},
{
desc: "One defautl and one TLD resolver, all doesn't resolve, returns error",
r: NewMultiResolver(
MultiResolverOptionWithResolver(doesntResolve, ""),
MultiResolverOptionWithResolver(doesntResolve, "eth"),
),
addr: ethAddr,
result: ethHash,
err: errors.New(`DNS name not found: "swarm.eth"`),
},
{
desc: "Two TLD resolvers, both resolve, returns resolved address",
r: NewMultiResolver(
MultiResolverOptionWithResolver(ethResolve, "eth"),
MultiResolverOptionWithResolver(testResolve, "test"),
),
addr: testAddr,
result: testHash,
},
{
desc: "One TLD resolver, no default resolver, returns error for different TLD",
r: NewMultiResolver(
MultiResolverOptionWithResolver(ethResolve, "eth"),
),
addr: testAddr,
err: NewNoResolverError("test"),
},
}
for _, x := range tests {
t.Run(x.desc, func(t *testing.T) {
res, err := x.r.Resolve(x.addr)
if err == nil {
if x.err != nil {
t.Fatalf("expected error %q, got result %q", x.err, res.Hex())
}
if res.Hex() != x.result {
t.Fatalf("expected result %q, got %q", x.result, res.Hex())
}
} else {
if x.err == nil {
t.Fatalf("expected no error, got %q", err)
}
if err.Error() != x.err.Error() {
t.Fatalf("expected error %q, got %q", x.err, err)
}
}
})
}
}
func TestDecryptOriginForbidden(t *testing.T) {
ctx := context.TODO()
ctx = sctx.SetHost(ctx, "swarm-gateways.net")
me := &ManifestEntry{
Access: &AccessEntry{Type: AccessTypePass},
}
api := NewAPI(nil, nil, nil, nil)
f := api.Decryptor(ctx, "")
err := f(me)
if err != ErrDecryptDomainForbidden {
t.Fatalf("should fail with ErrDecryptDomainForbidden, got %v", err)
}
}
func TestDecryptOrigin(t *testing.T) {
for _, v := range []struct {
host string
expectError error
}{
{
host: "localhost",
expectError: ErrDecrypt,
},
{
host: "127.0.0.1",
expectError: ErrDecrypt,
},
{
host: "swarm-gateways.net",
expectError: ErrDecryptDomainForbidden,
},
} {
ctx := context.TODO()
ctx = sctx.SetHost(ctx, v.host)
me := &ManifestEntry{
Access: &AccessEntry{Type: AccessTypePass},
}
api := NewAPI(nil, nil, nil, nil)
f := api.Decryptor(ctx, "")
err := f(me)
if err != v.expectError {
t.Fatalf("should fail with %v, got %v", v.expectError, err)
}
}
}
func TestDetectContentType(t *testing.T) {
for _, tc := range []struct {
file string
content string
expectedContentType string
}{
{
file: "file-with-correct-css.css",
content: "body {background-color: orange}",
expectedContentType: "text/css; charset=utf-8",
},
{
file: "empty-file.css",
content: "",
expectedContentType: "text/css; charset=utf-8",
},
{
file: "empty-file.pdf",
content: "",
expectedContentType: "application/pdf",
},
{
file: "empty-file.md",
content: "",
expectedContentType: "text/markdown; charset=utf-8",
},
{
file: "empty-file-with-unknown-content.strangeext",
content: "",
expectedContentType: "text/plain; charset=utf-8",
},
{
file: "file-with-unknown-extension-and-content.strangeext",
content: "Lorem Ipsum",
expectedContentType: "text/plain; charset=utf-8",
},
{
file: "file-no-extension",
content: "Lorem Ipsum",
expectedContentType: "text/plain; charset=utf-8",
},
{
file: "file-no-extension-no-content",
content: "",
expectedContentType: "text/plain; charset=utf-8",
},
{
file: "css-file-with-html-inside.css",
content: "<!doctype html><html><head></head><body></body></html>",
expectedContentType: "text/css; charset=utf-8",
},
} {
t.Run(tc.file, func(t *testing.T) {
detected, err := DetectContentType(tc.file, bytes.NewReader([]byte(tc.content)))
if err != nil {
t.Fatal(err)
}
if detected != tc.expectedContentType {
t.Fatalf("File: %s, Expected mime type %s, got %s", tc.file, tc.expectedContentType, detected)
}
})
}
}

797
swarm/api/client/client.go
View file

@ -1,797 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package client
import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/http/httptrace"
"net/textproto"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/swarm/api"
"github.com/ubiq/go-ubiq/swarm/spancontext"
"github.com/ubiq/go-ubiq/swarm/storage/feed"
"github.com/pborman/uuid"
)
var (
ErrUnauthorized = errors.New("unauthorized")
)
func NewClient(gateway string) *Client {
return &Client{
Gateway: gateway,
}
}
// Client wraps interaction with a swarm HTTP gateway.
type Client struct {
Gateway string
}
// UploadRaw uploads raw data to swarm and returns the resulting hash. If toEncrypt is true it
// uploads encrypted data
func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, error) {
if size <= 0 {
return "", errors.New("data size must be greater than zero")
}
addr := ""
if toEncrypt {
addr = "encrypt"
}
req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/"+addr, r)
if err != nil {
return "", err
}
req.ContentLength = size
res, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
return string(data), nil
}
// DownloadRaw downloads raw data from swarm and it returns a ReadCloser and a bool whether the
// content was encrypted
func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) {
uri := c.Gateway + "/bzz-raw:/" + hash
res, err := http.DefaultClient.Get(uri)
if err != nil {
return nil, false, err
}
if res.StatusCode != http.StatusOK {
res.Body.Close()
return nil, false, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
isEncrypted := (res.Header.Get("X-Decrypted") == "true")
return res.Body, isEncrypted, nil
}
// File represents a file in a swarm manifest and is used for uploading and
// downloading content to and from swarm
type File struct {
io.ReadCloser
api.ManifestEntry
}
// Open opens a local file which can then be passed to client.Upload to upload
// it to swarm
func Open(path string) (*File, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
stat, err := f.Stat()
if err != nil {
f.Close()
return nil, err
}
contentType, err := api.DetectContentType(f.Name(), f)
if err != nil {
return nil, err
}
return &File{
ReadCloser: f,
ManifestEntry: api.ManifestEntry{
ContentType: contentType,
Mode: int64(stat.Mode()),
Size: stat.Size(),
ModTime: stat.ModTime(),
},
}, nil
}
// Upload uploads a file to swarm and either adds it to an existing manifest
// (if the manifest argument is non-empty) or creates a new manifest containing
// the file, returning the resulting manifest hash (the file will then be
// available at bzz:/<hash>/<path>)
func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, error) {
if file.Size <= 0 {
return "", errors.New("file size must be greater than zero")
}
return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt)
}
// Download downloads a file with the given path from the swarm manifest with
// the given hash (i.e. it gets bzz:/<hash>/<path>)
func (c *Client) Download(hash, path string) (*File, error) {
uri := c.Gateway + "/bzz:/" + hash + "/" + path
res, err := http.DefaultClient.Get(uri)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
res.Body.Close()
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
return &File{
ReadCloser: res.Body,
ManifestEntry: api.ManifestEntry{
ContentType: res.Header.Get("Content-Type"),
Size: res.ContentLength,
},
}, nil
}
// UploadDirectory uploads a directory tree to swarm and either adds the files
// to an existing manifest (if the manifest argument is non-empty) or creates a
// new manifest, returning the resulting manifest hash (files from the
// directory will then be available at bzz:/<hash>/path/to/file), with
// the file specified in defaultPath being uploaded to the root of the manifest
// (i.e. bzz:/<hash>/)
func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bool) (string, error) {
stat, err := os.Stat(dir)
if err != nil {
return "", err
} else if !stat.IsDir() {
return "", fmt.Errorf("not a directory: %s", dir)
}
if defaultPath != "" {
if _, err := os.Stat(filepath.Join(dir, defaultPath)); err != nil {
if os.IsNotExist(err) {
return "", fmt.Errorf("the default path %q was not found in the upload directory %q", defaultPath, dir)
}
return "", fmt.Errorf("default path: %v", err)
}
}
return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt)
}
// DownloadDirectory downloads the files contained in a swarm manifest under
// the given path into a local directory (existing files will be overwritten)
func (c *Client) DownloadDirectory(hash, path, destDir, credentials string) error {
stat, err := os.Stat(destDir)
if err != nil {
return err
} else if !stat.IsDir() {
return fmt.Errorf("not a directory: %s", destDir)
}
uri := c.Gateway + "/bzz:/" + hash + "/" + path
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return err
}
if credentials != "" {
req.SetBasicAuth("", credentials)
}
req.Header.Set("Accept", "application/x-tar")
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusOK:
case http.StatusUnauthorized:
return ErrUnauthorized
default:
return fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
tr := tar.NewReader(res.Body)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil
} else if err != nil {
return err
}
// ignore the default path file
if hdr.Name == "" {
continue
}
dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path)))
if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
return err
}
var mode os.FileMode = 0644
if hdr.Mode > 0 {
mode = os.FileMode(hdr.Mode)
}
dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
if err != nil {
return err
}
n, err := io.Copy(dst, tr)
dst.Close()
if err != nil {
return err
} else if n != hdr.Size {
return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n)
}
}
}
// DownloadFile downloads a single file into the destination directory
// if the manifest entry does not specify a file name - it will fallback
// to the hash of the file as a filename
func (c *Client) DownloadFile(hash, path, dest, credentials string) error {
hasDestinationFilename := false
if stat, err := os.Stat(dest); err == nil {
hasDestinationFilename = !stat.IsDir()
} else {
if os.IsNotExist(err) {
// does not exist - should be created
hasDestinationFilename = true
} else {
return fmt.Errorf("could not stat path: %v", err)
}
}
manifestList, err := c.List(hash, path, credentials)
if err != nil {
return err
}
switch len(manifestList.Entries) {
case 0:
return fmt.Errorf("could not find path requested at manifest address. make sure the path you've specified is correct")
case 1:
//continue
default:
return fmt.Errorf("got too many matches for this path")
}
uri := c.Gateway + "/bzz:/" + hash + "/" + path
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return err
}
if credentials != "" {
req.SetBasicAuth("", credentials)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusOK:
case http.StatusUnauthorized:
return ErrUnauthorized
default:
return fmt.Errorf("unexpected HTTP status: expected 200 OK, got %d", res.StatusCode)
}
filename := ""
if hasDestinationFilename {
filename = dest
} else {
// try to assert
re := regexp.MustCompile("[^/]+$") //everything after last slash
if results := re.FindAllString(path, -1); len(results) > 0 {
filename = results[len(results)-1]
} else {
if entry := manifestList.Entries[0]; entry.Path != "" && entry.Path != "/" {
filename = entry.Path
} else {
// assume hash as name if there's nothing from the command line
filename = hash
}
}
filename = filepath.Join(dest, filename)
}
filePath, err := filepath.Abs(filename)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(filePath), 0777); err != nil {
return err
}
dst, err := os.Create(filename)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, res.Body)
return err
}
// UploadManifest uploads the given manifest to swarm
func (c *Client) UploadManifest(m *api.Manifest, toEncrypt bool) (string, error) {
data, err := json.Marshal(m)
if err != nil {
return "", err
}
return c.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt)
}
// DownloadManifest downloads a swarm manifest
func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) {
res, isEncrypted, err := c.DownloadRaw(hash)
if err != nil {
return nil, isEncrypted, err
}
defer res.Close()
var manifest api.Manifest
if err := json.NewDecoder(res).Decode(&manifest); err != nil {
return nil, isEncrypted, err
}
return &manifest, isEncrypted, nil
}
// List list files in a swarm manifest which have the given prefix, grouping
// common prefixes using "/" as a delimiter.
//
// For example, if the manifest represents the following directory structure:
//
// file1.txt
// file2.txt
// dir1/file3.txt
// dir1/dir2/file4.txt
//
// Then:
//
// - a prefix of "" would return [dir1/, file1.txt, file2.txt]
// - a prefix of "file" would return [file1.txt, file2.txt]
// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
//
// where entries ending with "/" are common prefixes.
func (c *Client) List(hash, prefix, credentials string) (*api.ManifestList, error) {
req, err := http.NewRequest(http.MethodGet, c.Gateway+"/bzz-list:/"+hash+"/"+prefix, nil)
if err != nil {
return nil, err
}
if credentials != "" {
req.SetBasicAuth("", credentials)
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer res.Body.Close()
switch res.StatusCode {
case http.StatusOK:
case http.StatusUnauthorized:
return nil, ErrUnauthorized
default:
return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
var list api.ManifestList
if err := json.NewDecoder(res.Body).Decode(&list); err != nil {
return nil, err
}
return &list, nil
}
// Uploader uploads files to swarm using a provided UploadFn
type Uploader interface {
Upload(UploadFn) error
}
type UploaderFunc func(UploadFn) error
func (u UploaderFunc) Upload(upload UploadFn) error {
return u(upload)
}
// DirectoryUploader uploads all files in a directory, optionally uploading
// a file to the default path
type DirectoryUploader struct {
Dir string
}
// Upload performs the upload of the directory and default path
func (d *DirectoryUploader) Upload(upload UploadFn) error {
return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
if f.IsDir() {
return nil
}
file, err := Open(path)
if err != nil {
return err
}
relPath, err := filepath.Rel(d.Dir, path)
if err != nil {
return err
}
file.Path = filepath.ToSlash(relPath)
return upload(file)
})
}
// FileUploader uploads a single file
type FileUploader struct {
File *File
}
// Upload performs the upload of the file
func (f *FileUploader) Upload(upload UploadFn) error {
return upload(f.File)
}
// UploadFn is the type of function passed to an Uploader to perform the upload
// of a single file (for example, a directory uploader would call a provided
// UploadFn for each file in the directory tree)
type UploadFn func(file *File) error
// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
// returning the resulting manifest hash
func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) {
ctx, sp := spancontext.StartSpan(context.Background(), "api.client.tarupload")
defer sp.Finish()
var tn time.Time
reqR, reqW := io.Pipe()
defer reqR.Close()
addr := hash
// If there is a hash already (a manifest), then that manifest will determine if the upload has
// to be encrypted or not. If there is no manifest then the toEncrypt parameter decides if
// there is encryption or not.
if hash == "" && toEncrypt {
// This is the built-in address for the encrypted upload endpoint
addr = "encrypt"
}
req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+addr, reqR)
if err != nil {
return "", err
}
trace := GetClientTrace("swarm api client - upload tar", "api.client.uploadtar", uuid.New()[:8], &tn)
req = req.WithContext(httptrace.WithClientTrace(ctx, trace))
transport := http.DefaultTransport
req.Header.Set("Content-Type", "application/x-tar")
if defaultPath != "" {
q := req.URL.Query()
q.Set("defaultpath", defaultPath)
req.URL.RawQuery = q.Encode()
}
// use 'Expect: 100-continue' so we don't send the request body if
// the server refuses the request
req.Header.Set("Expect", "100-continue")
tw := tar.NewWriter(reqW)
// define an UploadFn which adds files to the tar stream
uploadFn := func(file *File) error {
hdr := &tar.Header{
Name: file.Path,
Mode: file.Mode,
Size: file.Size,
ModTime: file.ModTime,
Xattrs: map[string]string{
"user.swarm.content-type": file.ContentType,
},
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
_, err = io.Copy(tw, file)
return err
}
// run the upload in a goroutine so we can send the request headers and
// wait for a '100 Continue' response before sending the tar stream
go func() {
err := uploader.Upload(uploadFn)
if err == nil {
err = tw.Close()
}
reqW.CloseWithError(err)
}()
tn = time.Now()
res, err := transport.RoundTrip(req)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
return string(data), nil
}
// MultipartUpload uses the given Uploader to upload files to swarm as a
// multipart form, returning the resulting manifest hash
func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) {
reqR, reqW := io.Pipe()
defer reqR.Close()
req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
if err != nil {
return "", err
}
// use 'Expect: 100-continue' so we don't send the request body if
// the server refuses the request
req.Header.Set("Expect", "100-continue")
mw := multipart.NewWriter(reqW)
req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
// define an UploadFn which adds files to the multipart form
uploadFn := func(file *File) error {
hdr := make(textproto.MIMEHeader)
hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path))
hdr.Set("Content-Type", file.ContentType)
hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10))
w, err := mw.CreatePart(hdr)
if err != nil {
return err
}
_, err = io.Copy(w, file)
return err
}
// run the upload in a goroutine so we can send the request headers and
// wait for a '100 Continue' response before sending the multipart form
go func() {
err := uploader.Upload(uploadFn)
if err == nil {
err = mw.Close()
}
reqW.CloseWithError(err)
}()
res, err := http.DefaultClient.Do(req)
if err != nil {
return "", err
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
}
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return "", err
}
return string(data), nil
}
// ErrNoFeedUpdatesFound is returned when Swarm cannot find updates of the given feed
var ErrNoFeedUpdatesFound = errors.New("No updates found for this feed")
// CreateFeedWithManifest creates a feed manifest, initializing it with the provided
// data
// Returns the resulting feed manifest address that you can use to include in an ENS Resolver (setContent)
// or reference future updates (Client.UpdateFeed)
func (c *Client) CreateFeedWithManifest(request *feed.Request) (string, error) {
responseStream, err := c.updateFeed(request, true)
if err != nil {
return "", err
}
defer responseStream.Close()
body, err := ioutil.ReadAll(responseStream)
if err != nil {
return "", err
}
var manifestAddress string
if err = json.Unmarshal(body, &manifestAddress); err != nil {
return "", err
}
return manifestAddress, nil
}
// UpdateFeed allows you to set a new version of your content
func (c *Client) UpdateFeed(request *feed.Request) error {
_, err := c.updateFeed(request, false)
return err
}
func (c *Client) updateFeed(request *feed.Request, createManifest bool) (io.ReadCloser, error) {
URL, err := url.Parse(c.Gateway)
if err != nil {
return nil, err
}
URL.Path = "/bzz-feed:/"
values := URL.Query()
body := request.AppendValues(values)
if createManifest {
values.Set("manifest", "1")
}
URL.RawQuery = values.Encode()
req, err := http.NewRequest("POST", URL.String(), bytes.NewBuffer(body))
if err != nil {
return nil, err
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
return res.Body, nil
}
// QueryFeed returns a byte stream with the raw content of the feed update
// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver
// points to that address
func (c *Client) QueryFeed(query *feed.Query, manifestAddressOrDomain string) (io.ReadCloser, error) {
return c.queryFeed(query, manifestAddressOrDomain, false)
}
// queryFeed returns a byte stream with the raw content of the feed update
// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver
// points to that address
// meta set to true will instruct the node return feed metainformation instead
func (c *Client) queryFeed(query *feed.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) {
URL, err := url.Parse(c.Gateway)
if err != nil {
return nil, err
}
URL.Path = "/bzz-feed:/" + manifestAddressOrDomain
values := URL.Query()
if query != nil {
query.AppendValues(values) //adds query parameters
}
if meta {
values.Set("meta", "1")
}
URL.RawQuery = values.Encode()
res, err := http.Get(URL.String())
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
if res.StatusCode == http.StatusNotFound {
return nil, ErrNoFeedUpdatesFound
}
errorMessageBytes, err := ioutil.ReadAll(res.Body)
var errorMessage string
if err != nil {
errorMessage = "cannot retrieve error message: " + err.Error()
} else {
errorMessage = string(errorMessageBytes)
}
return nil, fmt.Errorf("Error retrieving feed updates: %s", errorMessage)
}
return res.Body, nil
}
// GetFeedRequest returns a structure that describes the referenced feed status
// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver
// points to that address
func (c *Client) GetFeedRequest(query *feed.Query, manifestAddressOrDomain string) (*feed.Request, error) {
responseStream, err := c.queryFeed(query, manifestAddressOrDomain, true)
if err != nil {
return nil, err
}
defer responseStream.Close()
body, err := ioutil.ReadAll(responseStream)
if err != nil {
return nil, err
}
var metadata feed.Request
if err := metadata.UnmarshalJSON(body); err != nil {
return nil, err
}
return &metadata, nil
}
func GetClientTrace(traceMsg, metricPrefix, ruid string, tn *time.Time) *httptrace.ClientTrace {
trace := &httptrace.ClientTrace{
GetConn: func(_ string) {
log.Trace(traceMsg+" - http get", "event", "GetConn", "ruid", ruid)
metrics.GetOrRegisterResettingTimer(metricPrefix+".getconn", nil).Update(time.Since(*tn))
},
GotConn: func(_ httptrace.GotConnInfo) {
log.Trace(traceMsg+" - http get", "event", "GotConn", "ruid", ruid)
metrics.GetOrRegisterResettingTimer(metricPrefix+".gotconn", nil).Update(time.Since(*tn))
},
PutIdleConn: func(err error) {
log.Trace(traceMsg+" - http get", "event", "PutIdleConn", "ruid", ruid, "err", err)
metrics.GetOrRegisterResettingTimer(metricPrefix+".putidle", nil).Update(time.Since(*tn))
},
GotFirstResponseByte: func() {
log.Trace(traceMsg+" - http get", "event", "GotFirstResponseByte", "ruid", ruid)
metrics.GetOrRegisterResettingTimer(metricPrefix+".firstbyte", nil).Update(time.Since(*tn))
},
Got100Continue: func() {
log.Trace(traceMsg, "event", "Got100Continue", "ruid", ruid)
metrics.GetOrRegisterResettingTimer(metricPrefix+".got100continue", nil).Update(time.Since(*tn))
},
DNSStart: func(_ httptrace.DNSStartInfo) {
log.Trace(traceMsg, "event", "DNSStart", "ruid", ruid)
metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsstart", nil).Update(time.Since(*tn))
},
DNSDone: func(_ httptrace.DNSDoneInfo) {
log.Trace(traceMsg, "event", "DNSDone", "ruid", ruid)
metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsdone", nil).Update(time.Since(*tn))
},
ConnectStart: func(network, addr string) {
log.Trace(traceMsg, "event", "ConnectStart", "ruid", ruid, "network", network, "addr", addr)
metrics.GetOrRegisterResettingTimer(metricPrefix+".connectstart", nil).Update(time.Since(*tn))
},
ConnectDone: func(network, addr string, err error) {
log.Trace(traceMsg, "event", "ConnectDone", "ruid", ruid, "network", network, "addr", addr, "err", err)
metrics.GetOrRegisterResettingTimer(metricPrefix+".connectdone", nil).Update(time.Since(*tn))
},
WroteHeaders: func() {
log.Trace(traceMsg, "event", "WroteHeaders(request)", "ruid", ruid)
metrics.GetOrRegisterResettingTimer(metricPrefix+".wroteheaders", nil).Update(time.Since(*tn))
},
Wait100Continue: func() {
log.Trace(traceMsg, "event", "Wait100Continue", "ruid", ruid)
metrics.GetOrRegisterResettingTimer(metricPrefix+".wait100continue", nil).Update(time.Since(*tn))
},
WroteRequest: func(_ httptrace.WroteRequestInfo) {
log.Trace(traceMsg, "event", "WroteRequest", "ruid", ruid)
metrics.GetOrRegisterResettingTimer(metricPrefix+".wroterequest", nil).Update(time.Since(*tn))
},
}
return trace
}

589
swarm/api/client/client_test.go
View file

@ -1,589 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package client
import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"sort"
"testing"
"github.com/ubiq/go-ubiq/swarm/storage"
"github.com/ubiq/go-ubiq/swarm/storage/feed/lookup"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/crypto"
"github.com/ubiq/go-ubiq/swarm/api"
swarmhttp "github.com/ubiq/go-ubiq/swarm/api/http"
"github.com/ubiq/go-ubiq/swarm/storage/feed"
)
func serverFunc(api *api.API) swarmhttp.TestServer {
return swarmhttp.NewServer(api, "")
}
// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
func TestClientUploadDownloadRaw(t *testing.T) {
testClientUploadDownloadRaw(false, t)
}
func TestClientUploadDownloadRawEncrypted(t *testing.T) {
testClientUploadDownloadRaw(true, t)
}
func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) {
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
client := NewClient(srv.URL)
// upload some raw data
data := []byte("foo123")
hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt)
if err != nil {
t.Fatal(err)
}
// check we can download the same data
res, isEncrypted, err := client.DownloadRaw(hash)
if err != nil {
t.Fatal(err)
}
if isEncrypted != toEncrypt {
t.Fatalf("Expected encyption status %v got %v", toEncrypt, isEncrypted)
}
defer res.Close()
gotData, err := ioutil.ReadAll(res)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(gotData, data) {
t.Fatalf("expected downloaded data to be %q, got %q", data, gotData)
}
}
// TestClientUploadDownloadFiles test uploading and downloading files to swarm
// manifests
func TestClientUploadDownloadFiles(t *testing.T) {
testClientUploadDownloadFiles(false, t)
}
func TestClientUploadDownloadFilesEncrypted(t *testing.T) {
testClientUploadDownloadFiles(true, t)
}
func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) {
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
client := NewClient(srv.URL)
upload := func(manifest, path string, data []byte) string {
file := &File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
ManifestEntry: api.ManifestEntry{
Path: path,
ContentType: "text/plain",
Size: int64(len(data)),
},
}
hash, err := client.Upload(file, manifest, toEncrypt)
if err != nil {
t.Fatal(err)
}
return hash
}
checkDownload := func(manifest, path string, expected []byte) {
file, err := client.Download(manifest, path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
if file.Size != int64(len(expected)) {
t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size)
}
if file.ContentType != "text/plain" {
t.Fatalf("expected downloaded file to have type %q, got %q", "text/plain", file.ContentType)
}
data, err := ioutil.ReadAll(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, expected) {
t.Fatalf("expected downloaded data to be %q, got %q", expected, data)
}
}
// upload a file to the root of a manifest
rootData := []byte("some-data")
rootHash := upload("", "", rootData)
// check we can download the root file
checkDownload(rootHash, "", rootData)
// upload another file to the same manifest
otherData := []byte("some-other-data")
newHash := upload(rootHash, "some/other/path", otherData)
// check we can download both files from the new manifest
checkDownload(newHash, "", rootData)
checkDownload(newHash, "some/other/path", otherData)
// replace the root file with different data
newHash = upload(newHash, "", otherData)
// check both files have the other data
checkDownload(newHash, "", otherData)
checkDownload(newHash, "some/other/path", otherData)
}
var testDirFiles = []string{
"file1.txt",
"file2.txt",
"dir1/file3.txt",
"dir1/file4.txt",
"dir2/file5.txt",
"dir2/dir3/file6.txt",
"dir2/dir4/file7.txt",
"dir2/dir4/file8.txt",
}
func newTestDirectory(t *testing.T) string {
dir, err := ioutil.TempDir("", "swarm-client-test")
if err != nil {
t.Fatal(err)
}
for _, file := range testDirFiles {
path := filepath.Join(dir, file)
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
os.RemoveAll(dir)
t.Fatalf("error creating dir for %s: %s", path, err)
}
if err := ioutil.WriteFile(path, []byte(file), 0644); err != nil {
os.RemoveAll(dir)
t.Fatalf("error writing file %s: %s", path, err)
}
}
return dir
}
// TestClientUploadDownloadDirectory tests uploading and downloading a
// directory of files to a swarm manifest
func TestClientUploadDownloadDirectory(t *testing.T) {
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
dir := newTestDirectory(t)
defer os.RemoveAll(dir)
// upload the directory
client := NewClient(srv.URL)
defaultPath := testDirFiles[0]
hash, err := client.UploadDirectory(dir, defaultPath, "", false)
if err != nil {
t.Fatalf("error uploading directory: %s", err)
}
// check we can download the individual files
checkDownloadFile := func(path string, expected []byte) {
file, err := client.Download(hash, path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
data, err := ioutil.ReadAll(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, expected) {
t.Fatalf("expected data to be %q, got %q", expected, data)
}
}
for _, file := range testDirFiles {
checkDownloadFile(file, []byte(file))
}
// check we can download the default path
checkDownloadFile("", []byte(testDirFiles[0]))
// check we can download the directory
tmp, err := ioutil.TempDir("", "swarm-client-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmp)
if err := client.DownloadDirectory(hash, "", tmp, ""); err != nil {
t.Fatal(err)
}
for _, file := range testDirFiles {
data, err := ioutil.ReadFile(filepath.Join(tmp, file))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, []byte(file)) {
t.Fatalf("expected data to be %q, got %q", file, data)
}
}
}
// TestClientFileList tests listing files in a swarm manifest
func TestClientFileList(t *testing.T) {
testClientFileList(false, t)
}
func TestClientFileListEncrypted(t *testing.T) {
testClientFileList(true, t)
}
func testClientFileList(toEncrypt bool, t *testing.T) {
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
dir := newTestDirectory(t)
defer os.RemoveAll(dir)
client := NewClient(srv.URL)
hash, err := client.UploadDirectory(dir, "", "", toEncrypt)
if err != nil {
t.Fatalf("error uploading directory: %s", err)
}
ls := func(prefix string) []string {
list, err := client.List(hash, prefix, "")
if err != nil {
t.Fatal(err)
}
paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries))
paths = append(paths, list.CommonPrefixes...)
for _, entry := range list.Entries {
paths = append(paths, entry.Path)
}
sort.Strings(paths)
return paths
}
tests := map[string][]string{
"": {"dir1/", "dir2/", "file1.txt", "file2.txt"},
"file": {"file1.txt", "file2.txt"},
"file1": {"file1.txt"},
"file2.txt": {"file2.txt"},
"file12": {},
"dir": {"dir1/", "dir2/"},
"dir1": {"dir1/"},
"dir1/": {"dir1/file3.txt", "dir1/file4.txt"},
"dir1/file": {"dir1/file3.txt", "dir1/file4.txt"},
"dir1/file3.txt": {"dir1/file3.txt"},
"dir1/file34": {},
"dir2/": {"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"},
"dir2/file": {"dir2/file5.txt"},
"dir2/dir": {"dir2/dir3/", "dir2/dir4/"},
"dir2/dir3/": {"dir2/dir3/file6.txt"},
"dir2/dir4/": {"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
"dir2/dir4/file": {"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
"dir2/dir4/file7.txt": {"dir2/dir4/file7.txt"},
"dir2/dir4/file78": {},
}
for prefix, expected := range tests {
actual := ls(prefix)
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual)
}
}
}
// TestClientMultipartUpload tests uploading files to swarm using a multipart
// upload
func TestClientMultipartUpload(t *testing.T) {
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
// define an uploader which uploads testDirFiles with some data
data := []byte("some-data")
uploader := UploaderFunc(func(upload UploadFn) error {
for _, name := range testDirFiles {
file := &File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
ManifestEntry: api.ManifestEntry{
Path: name,
ContentType: "text/plain",
Size: int64(len(data)),
},
}
if err := upload(file); err != nil {
return err
}
}
return nil
})
// upload the files as a multipart upload
client := NewClient(srv.URL)
hash, err := client.MultipartUpload("", uploader)
if err != nil {
t.Fatal(err)
}
// check we can download the individual files
checkDownloadFile := func(path string) {
file, err := client.Download(hash, path)
if err != nil {
t.Fatal(err)
}
defer file.Close()
gotData, err := ioutil.ReadAll(file)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(gotData, data) {
t.Fatalf("expected data to be %q, got %q", data, gotData)
}
}
for _, file := range testDirFiles {
checkDownloadFile(file)
}
}
func newTestSigner() (*feed.GenericSigner, error) {
privKey, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
if err != nil {
return nil, err
}
return feed.NewGenericSigner(privKey), nil
}
// Test the transparent resolving of feed updates with bzz:// scheme
//
// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update.
// This effectively uses a feed to store a pointer to content rather than the content itself
// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data
// and raw retrieve of that hash should return the data
func TestClientBzzWithFeed(t *testing.T) {
signer, _ := newTestSigner()
// Initialize a Swarm test server
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
swarmClient := NewClient(srv.URL)
defer srv.Close()
// put together some data for our test:
dataBytes := []byte(`
//
// Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update.
// So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it:
//
// MANIFEST HASH --> DATA
//
// Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this,
// we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash.
//
// FEED MANIFEST HASH --> MANIFEST HASH --> DATA
//
// Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash**
// stays constant, we have effectively created a fixed address to changing content. (Applause)
//
// FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2)
//
`)
// Create a virtual File out of memory containing the above data
f := &File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)),
ManifestEntry: api.ManifestEntry{
ContentType: "text/plain",
Mode: 0660,
Size: int64(len(dataBytes)),
},
}
// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
manifestAddressHex, err := swarmClient.Upload(f, "", false)
if err != nil {
t.Fatalf("Error creating manifest: %s", err)
}
// convert the hex-encoded manifest hash to a 32-byte slice
manifestAddress := common.FromHex(manifestAddressHex)
if len(manifestAddress) != storage.AddressLength {
t.Fatalf("Something went wrong. Got a hash of an unexpected length. Expected %d bytes. Got %d", storage.AddressLength, len(manifestAddress))
}
// Now create a **feed manifest**. For that, we need a topic:
topic, _ := feed.NewTopic("interesting topic indeed", nil)
// Build a feed request to update data
request := feed.NewFirstRequest(topic)
// Put the 32-byte address of the manifest into the feed update
request.SetData(manifestAddress)
// Sign the update
if err := request.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}
// Publish the update and at the same time request a **feed manifest** to be created
feedManifestAddressHex, err := swarmClient.CreateFeedWithManifest(request)
if err != nil {
t.Fatalf("Error creating feed manifest: %s", err)
}
// Check we have received the exact **feed manifest** to be expected
// given the topic and user signing the updates:
correctFeedManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2"
if feedManifestAddressHex != correctFeedManifestAddrHex {
t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctFeedManifestAddrHex, feedManifestAddressHex)
}
// Check we get a not found error when trying to get feed updates with a made-up manifest
_, err = swarmClient.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
if err != ErrNoFeedUpdatesFound {
t.Fatalf("Expected to receive ErrNoFeedUpdatesFound error. Got: %s", err)
}
// If we query the feed directly we should get **manifest hash** back:
reader, err := swarmClient.QueryFeed(nil, correctFeedManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving feed updates: %s", err)
}
defer reader.Close()
gotData, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
//Check that indeed the **manifest hash** is retrieved
if !bytes.Equal(manifestAddress, gotData) {
t.Fatalf("Expected: %v, got %v", manifestAddress, gotData)
}
// Now the final test we were looking for: Use bzz://<feed-manifest> and that should resolve all manifests
// and return the original data directly:
f, err = swarmClient.Download(feedManifestAddressHex, "")
if err != nil {
t.Fatal(err)
}
gotData, err = ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}
// Check that we get back the original data:
if !bytes.Equal(dataBytes, gotData) {
t.Fatalf("Expected: %v, got %v", manifestAddress, gotData)
}
}
// TestClientCreateUpdateFeed will check that feeds can be created and updated via the HTTP client.
func TestClientCreateUpdateFeed(t *testing.T) {
signer, _ := newTestSigner()
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
client := NewClient(srv.URL)
defer srv.Close()
// set raw data for the feed update
databytes := []byte("En un lugar de La Mancha, de cuyo nombre no quiero acordarme...")
// our feed topic name
topic, _ := feed.NewTopic("El Quijote", nil)
createRequest := feed.NewFirstRequest(topic)
createRequest.SetData(databytes)
if err := createRequest.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}
feedManifestHash, err := client.CreateFeedWithManifest(createRequest)
if err != nil {
t.Fatal(err)
}
correctManifestAddrHex := "0e9b645ebc3da167b1d56399adc3276f7a08229301b72a03336be0e7d4b71882"
if feedManifestHash != correctManifestAddrHex {
t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash)
}
reader, err := client.QueryFeed(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving feed updates: %s", err)
}
defer reader.Close()
gotData, err := ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(databytes, gotData) {
t.Fatalf("Expected: %v, got %v", databytes, gotData)
}
// define different data
databytes = []byte("... no ha mucho tiempo que vivía un hidalgo de los de lanza en astillero ...")
updateRequest, err := client.GetFeedRequest(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving update request template: %s", err)
}
updateRequest.SetData(databytes)
if err := updateRequest.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}
if err = client.UpdateFeed(updateRequest); err != nil {
t.Fatalf("Error updating feed: %s", err)
}
reader, err = client.QueryFeed(nil, correctManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving feed updates: %s", err)
}
defer reader.Close()
gotData, err = ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(databytes, gotData) {
t.Fatalf("Expected: %v, got %v", databytes, gotData)
}
// now try retrieving feed updates without a manifest
fd := &feed.Feed{
Topic: topic,
User: signer.Address(),
}
lookupParams := feed.NewQueryLatest(fd, lookup.NoClue)
reader, err = client.QueryFeed(lookupParams, "")
if err != nil {
t.Fatalf("Error retrieving feed updates: %s", err)
}
defer reader.Close()
gotData, err = ioutil.ReadAll(reader)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(databytes, gotData) {
t.Fatalf("Expected: %v, got %v", databytes, gotData)
}
}

143
swarm/api/config.go
View file

@ -1,143 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"crypto/ecdsa"
"fmt"
"os"
"path/filepath"
"time"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/contracts/ens"
"github.com/ubiq/go-ubiq/crypto"
"github.com/ubiq/go-ubiq/node"
"github.com/ubiq/go-ubiq/p2p/enode"
"github.com/ubiq/go-ubiq/swarm/log"
"github.com/ubiq/go-ubiq/swarm/network"
"github.com/ubiq/go-ubiq/swarm/pss"
"github.com/ubiq/go-ubiq/swarm/services/swap"
"github.com/ubiq/go-ubiq/swarm/storage"
)
const (
DefaultHTTPListenAddr = "127.0.0.1"
DefaultHTTPPort = "8500"
)
// separate bzz directories
// allow several bzz nodes running in parallel
type Config struct {
// serialised/persisted fields
*storage.FileStoreParams
*storage.LocalStoreParams
*network.HiveParams
Swap *swap.LocalProfile
Pss *pss.PssParams
//*network.SyncParams
Contract common.Address
EnsRoot common.Address
EnsAPIs []string
Path string
ListenAddr string
Port string
PublicKey string
BzzKey string
NodeID string
NetworkID uint64
SwapEnabled bool
SyncEnabled bool
SyncingSkipCheck bool
DeliverySkipCheck bool
MaxStreamPeerServers int
LightNodeEnabled bool
BootnodeMode bool
SyncUpdateDelay time.Duration
SwapAPI string
Cors string
BzzAccount string
GlobalStoreAPI string
privateKey *ecdsa.PrivateKey
}
//create a default config with all parameters to set to defaults
func NewConfig() (c *Config) {
c = &Config{
LocalStoreParams: storage.NewDefaultLocalStoreParams(),
FileStoreParams: storage.NewFileStoreParams(),
HiveParams: network.NewHiveParams(),
//SyncParams: network.NewDefaultSyncParams(),
Swap: swap.NewDefaultSwapParams(),
Pss: pss.NewPssParams(),
ListenAddr: DefaultHTTPListenAddr,
Port: DefaultHTTPPort,
Path: node.DefaultDataDir(),
EnsAPIs: nil,
EnsRoot: ens.TestNetAddress,
NetworkID: network.DefaultNetworkID,
SwapEnabled: false,
SyncEnabled: true,
SyncingSkipCheck: false,
MaxStreamPeerServers: 10000,
DeliverySkipCheck: true,
SyncUpdateDelay: 15 * time.Second,
SwapAPI: "",
}
return
}
//some config params need to be initialized after the complete
//config building phase is completed (e.g. due to overriding flags)
func (c *Config) Init(prvKey *ecdsa.PrivateKey) {
address := crypto.PubkeyToAddress(prvKey.PublicKey)
c.Path = filepath.Join(c.Path, "bzz-"+common.Bytes2Hex(address.Bytes()))
err := os.MkdirAll(c.Path, os.ModePerm)
if err != nil {
log.Error(fmt.Sprintf("Error creating root swarm data directory: %v", err))
return
}
pubkey := crypto.FromECDSAPub(&prvKey.PublicKey)
pubkeyhex := common.ToHex(pubkey)
keyhex := crypto.Keccak256Hash(pubkey).Hex()
c.PublicKey = pubkeyhex
c.BzzKey = keyhex
c.NodeID = enode.PubkeyToIDV4(&prvKey.PublicKey).String()
if c.SwapEnabled {
c.Swap.Init(c.Contract, prvKey)
}
c.privateKey = prvKey
c.LocalStoreParams.Init(c.Path)
c.LocalStoreParams.BaseKey = common.FromHex(keyhex)
c.Pss = c.Pss.WithPrivateKey(c.privateKey)
}
func (c *Config) ShiftPrivateKey() (privKey *ecdsa.PrivateKey) {
if c.privateKey != nil {
privKey = c.privateKey
c.privateKey = nil
}
return privKey
}

59
swarm/api/config_test.go
View file

@ -1,59 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"reflect"
"testing"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/crypto"
)
func TestConfig(t *testing.T) {
var hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c"
prvkey, err := crypto.HexToECDSA(hexprvkey)
if err != nil {
t.Fatalf("failed to load private key: %v", err)
}
one := NewConfig()
two := NewConfig()
one.LocalStoreParams = two.LocalStoreParams
if equal := reflect.DeepEqual(one, two); !equal {
t.Fatal("Two default configs are not equal")
}
one.Init(prvkey)
//the init function should set the following fields
if one.BzzKey == "" {
t.Fatal("Expected BzzKey to be set")
}
if one.PublicKey == "" {
t.Fatal("Expected PublicKey to be set")
}
if one.Swap.PayProfile.Beneficiary == (common.Address{}) && one.SwapEnabled {
t.Fatal("Failed to correctly initialize SwapParams")
}
if one.ChunkDbPath == one.Path {
t.Fatal("Failed to correctly initialize StoreParams")
}
}

78
swarm/api/encrypt.go
View file

@ -1,78 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"encoding/binary"
"errors"
"github.com/ubiq/go-ubiq/swarm/storage/encryption"
"golang.org/x/crypto/sha3"
)
type RefEncryption struct {
refSize int
span []byte
}
func NewRefEncryption(refSize int) *RefEncryption {
span := make([]byte, 8)
binary.LittleEndian.PutUint64(span, uint64(refSize))
return &RefEncryption{
refSize: refSize,
span: span,
}
}
func (re *RefEncryption) Encrypt(ref []byte, key []byte) ([]byte, error) {
spanEncryption := encryption.New(key, 0, uint32(re.refSize/32), sha3.NewLegacyKeccak256)
encryptedSpan, err := spanEncryption.Encrypt(re.span)
if err != nil {
return nil, err
}
dataEncryption := encryption.New(key, re.refSize, 0, sha3.NewLegacyKeccak256)
encryptedData, err := dataEncryption.Encrypt(ref)
if err != nil {
return nil, err
}
encryptedRef := make([]byte, len(ref)+8)
copy(encryptedRef[:8], encryptedSpan)
copy(encryptedRef[8:], encryptedData)
return encryptedRef, nil
}
func (re *RefEncryption) Decrypt(ref []byte, key []byte) ([]byte, error) {
spanEncryption := encryption.New(key, 0, uint32(re.refSize/32), sha3.NewLegacyKeccak256)
decryptedSpan, err := spanEncryption.Decrypt(ref[:8])
if err != nil {
return nil, err
}
size := binary.LittleEndian.Uint64(decryptedSpan)
if size != uint64(len(ref)-8) {
return nil, errors.New("invalid span in encrypted reference")
}
dataEncryption := encryption.New(key, re.refSize, 0, sha3.NewLegacyKeccak256)
decryptedRef, err := dataEncryption.Decrypt(ref[8:])
if err != nil {
return nil, err
}
return decryptedRef, nil
}

292
swarm/api/filesystem.go
View file

@ -1,292 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"bufio"
"context"
"fmt"
"io"
"os"
"path"
"path/filepath"
"sync"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/swarm/log"
"github.com/ubiq/go-ubiq/swarm/storage"
)
const maxParallelFiles = 5
type FileSystem struct {
api *API
}
func NewFileSystem(api *API) *FileSystem {
return &FileSystem{api}
}
// Upload replicates a local directory as a manifest file and uploads it
// using FileStore store
// This function waits the chunks to be stored.
// TODO: localpath should point to a manifest
//
// DEPRECATED: Use the HTTP API instead
func (fs *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error) {
var list []*manifestTrieEntry
localpath, err := filepath.Abs(filepath.Clean(lpath))
if err != nil {
return "", err
}
f, err := os.Open(localpath)
if err != nil {
return "", err
}
stat, err := f.Stat()
if err != nil {
return "", err
}
var start int
if stat.IsDir() {
start = len(localpath)
log.Debug(fmt.Sprintf("uploading '%s'", localpath))
err = filepath.Walk(localpath, func(path string, info os.FileInfo, err error) error {
if (err == nil) && !info.IsDir() {
if len(path) <= start {
return fmt.Errorf("Path is too short")
}
if path[:start] != localpath {
return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath)
}
entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(path)}, nil)
list = append(list, entry)
}
return err
})
if err != nil {
return "", err
}
} else {
dir := filepath.Dir(localpath)
start = len(dir)
if len(localpath) <= start {
return "", fmt.Errorf("Path is too short")
}
if localpath[:start] != dir {
return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir)
}
entry := newManifestTrieEntry(&ManifestEntry{Path: filepath.ToSlash(localpath)}, nil)
list = append(list, entry)
}
errors := make([]error, len(list))
sem := make(chan bool, maxParallelFiles)
defer close(sem)
for i, entry := range list {
sem <- true
go func(i int, entry *manifestTrieEntry) {
defer func() { <-sem }()
f, err := os.Open(entry.Path)
if err != nil {
errors[i] = err
return
}
defer f.Close()
stat, err := f.Stat()
if err != nil {
errors[i] = err
return
}
var hash storage.Address
var wait func(context.Context) error
ctx := context.TODO()
hash, wait, err = fs.api.fileStore.Store(ctx, f, stat.Size(), toEncrypt)
if err != nil {
errors[i] = err
return
}
if hash != nil {
list[i].Hash = hash.Hex()
}
if err := wait(ctx); err != nil {
errors[i] = err
return
}
list[i].ContentType, err = DetectContentType(f.Name(), f)
if err != nil {
errors[i] = err
return
}
}(i, entry)
}
for i := 0; i < cap(sem); i++ {
sem <- true
}
trie := &manifestTrie{
fileStore: fs.api.fileStore,
}
quitC := make(chan bool)
for i, entry := range list {
if errors[i] != nil {
return "", errors[i]
}
entry.Path = RegularSlashes(entry.Path[start:])
if entry.Path == index {
ientry := newManifestTrieEntry(&ManifestEntry{
ContentType: entry.ContentType,
}, nil)
ientry.Hash = entry.Hash
trie.addEntry(ientry, quitC)
}
trie.addEntry(entry, quitC)
}
err2 := trie.recalcAndStore()
var hs string
if err2 == nil {
hs = trie.ref.Hex()
}
return hs, err2
}
// Download replicates the manifest basePath structure on the local filesystem
// under localpath
//
// DEPRECATED: Use the HTTP API instead
func (fs *FileSystem) Download(bzzpath, localpath string) error {
lpath, err := filepath.Abs(filepath.Clean(localpath))
if err != nil {
return err
}
err = os.MkdirAll(lpath, os.ModePerm)
if err != nil {
return err
}
//resolving host and port
uri, err := Parse(path.Join("bzz:/", bzzpath))
if err != nil {
return err
}
addr, err := fs.api.Resolve(context.TODO(), uri.Addr)
if err != nil {
return err
}
path := uri.Path
if len(path) > 0 {
path += "/"
}
quitC := make(chan bool)
trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC, NOOPDecrypt)
if err != nil {
log.Warn(fmt.Sprintf("fs.Download: loadManifestTrie error: %v", err))
return err
}
type downloadListEntry struct {
addr storage.Address
path string
}
var list []*downloadListEntry
var mde error
prevPath := lpath
err = trie.listWithPrefix(path, quitC, func(entry *manifestTrieEntry, suffix string) {
log.Trace(fmt.Sprintf("fs.Download: %#v", entry))
addr = common.Hex2Bytes(entry.Hash)
path := lpath + "/" + suffix
dir := filepath.Dir(path)
if dir != prevPath {
mde = os.MkdirAll(dir, os.ModePerm)
prevPath = dir
}
if (mde == nil) && (path != dir+"/") {
list = append(list, &downloadListEntry{addr: addr, path: path})
}
})
if err != nil {
return err
}
wg := sync.WaitGroup{}
errC := make(chan error)
done := make(chan bool, maxParallelFiles)
for i, entry := range list {
select {
case done <- true:
wg.Add(1)
case <-quitC:
return fmt.Errorf("aborted")
}
go func(i int, entry *downloadListEntry) {
defer wg.Done()
err := retrieveToFile(quitC, fs.api.fileStore, entry.addr, entry.path)
if err != nil {
select {
case errC <- err:
case <-quitC:
}
return
}
<-done
}(i, entry)
}
go func() {
wg.Wait()
close(errC)
}()
select {
case err = <-errC:
return err
case <-quitC:
return fmt.Errorf("aborted")
}
}
func retrieveToFile(quitC chan bool, fileStore *storage.FileStore, addr storage.Address, path string) error {
f, err := os.Create(path) // TODO: basePath separators
if err != nil {
return err
}
reader, _ := fileStore.Retrieve(context.TODO(), addr)
writer := bufio.NewWriter(f)
size, err := reader.Size(context.TODO(), quitC)
if err != nil {
return err
}
if _, err = io.CopyN(writer, reader, size); err != nil {
return err
}
if err := writer.Flush(); err != nil {
return err
}
return f.Close()
}

199
swarm/api/filesystem_test.go
View file

@ -1,199 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"bytes"
"context"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/swarm/storage"
)
var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test")
func testFileSystem(t *testing.T, f func(*FileSystem, bool)) {
testAPI(t, func(api *API, toEncrypt bool) {
f(NewFileSystem(api), toEncrypt)
})
}
func readPath(t *testing.T, parts ...string) string {
file := filepath.Join(parts...)
content, err := ioutil.ReadFile(file)
if err != nil {
t.Fatalf("unexpected error reading '%v': %v", file, err)
}
return string(content)
}
func TestApiDirUpload0(t *testing.T) {
testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "", toEncrypt)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "index.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
content = readPath(t, "testdata", "test0", "index.css")
resp = testGet(t, api, bzzhash, "index.css")
exp = expResponse(content, "text/css; charset=utf-8", 0)
checkResponse(t, resp, exp)
addr := storage.Address(common.Hex2Bytes(bzzhash))
_, _, _, _, err = api.Get(context.TODO(), NOOPDecrypt, addr, "")
if err == nil {
t.Fatalf("expected error: %v", err)
}
downloadDir := filepath.Join(testDownloadDir, "test0")
defer os.RemoveAll(downloadDir)
err = fs.Download(bzzhash, downloadDir)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
newbzzhash, err := fs.Upload(downloadDir, "", toEncrypt)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// TODO: currently the hash is not deterministic in the encrypted case
if !toEncrypt && bzzhash != newbzzhash {
t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash)
}
})
}
func TestApiDirUploadModify(t *testing.T) {
testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "", toEncrypt)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
addr := storage.Address(common.Hex2Bytes(bzzhash))
addr, err = api.Modify(context.TODO(), addr, "index.html", "", "")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
index, err := ioutil.ReadFile(filepath.Join("testdata", "test0", "index.html"))
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
ctx := context.TODO()
hash, wait, err := api.Store(ctx, bytes.NewReader(index), int64(len(index)), toEncrypt)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
err = wait(ctx)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
addr, err = api.Modify(context.TODO(), addr, "index2.html", hash.Hex(), "text/html; charset=utf-8")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
addr, err = api.Modify(context.TODO(), addr, "img/logo.png", hash.Hex(), "text/html; charset=utf-8")
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
bzzhash = addr.Hex()
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "index2.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
resp = testGet(t, api, bzzhash, "img/logo.png")
exp = expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
content = readPath(t, "testdata", "test0", "index.css")
resp = testGet(t, api, bzzhash, "index.css")
exp = expResponse(content, "text/css; charset=utf-8", 0)
checkResponse(t, resp, exp)
_, _, _, _, err = api.Get(context.TODO(), nil, addr, "")
if err == nil {
t.Errorf("expected error: %v", err)
}
})
}
func TestApiDirUploadWithRootFile(t *testing.T) {
testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "index.html", toEncrypt)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
}
func TestApiFileUpload(t *testing.T) {
testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "", toEncrypt)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "index.html")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
}
func TestApiFileUploadWithRootFile(t *testing.T) {
testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
api := fs.api
bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "index.html", toEncrypt)
if err != nil {
t.Errorf("unexpected error: %v", err)
return
}
content := readPath(t, "testdata", "test0", "index.html")
resp := testGet(t, api, bzzhash, "")
exp := expResponse(content, "text/html; charset=utf-8", 0)
checkResponse(t, resp, exp)
})
}

1201
swarm/api/gen_mime.go
View file

File diff suppressed because it is too large Load diff

113
swarm/api/http/middleware.go
View file

@ -1,113 +0,0 @@
package http
import (
"fmt"
"net/http"
"runtime/debug"
"strings"
"time"
"github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/swarm/api"
"github.com/ubiq/go-ubiq/swarm/log"
"github.com/ubiq/go-ubiq/swarm/sctx"
"github.com/ubiq/go-ubiq/swarm/spancontext"
"github.com/pborman/uuid"
)
// Adapt chains h (main request handler) main handler to adapters (middleware handlers)
// Please note that the order of execution for `adapters` is FIFO (adapters[0] will be executed first)
func Adapt(h http.Handler, adapters ...Adapter) http.Handler {
for i := range adapters {
adapter := adapters[len(adapters)-1-i]
h = adapter(h)
}
return h
}
type Adapter func(http.Handler) http.Handler
func SetRequestID(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r = r.WithContext(SetRUID(r.Context(), uuid.New()[:8]))
metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
log.Info("created ruid for request", "ruid", GetRUID(r.Context()), "method", r.Method, "url", r.RequestURI)
h.ServeHTTP(w, r)
})
}
func SetRequestHost(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r = r.WithContext(sctx.SetHost(r.Context(), r.Host))
log.Info("setting request host", "ruid", GetRUID(r.Context()), "host", sctx.GetHost(r.Context()))
h.ServeHTTP(w, r)
})
}
func ParseURI(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
if err != nil {
w.WriteHeader(http.StatusBadRequest)
respondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
return
}
if uri.Addr != "" && strings.HasPrefix(uri.Addr, "0x") {
uri.Addr = strings.TrimPrefix(uri.Addr, "0x")
msg := fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
Please click <a href='%[1]s'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uri.String())
w.WriteHeader(http.StatusNotFound)
w.Write([]byte(msg))
return
}
ctx := r.Context()
r = r.WithContext(SetURI(ctx, uri))
log.Debug("parsed request path", "ruid", GetRUID(r.Context()), "method", r.Method, "uri.Addr", uri.Addr, "uri.Path", uri.Path, "uri.Scheme", uri.Scheme)
h.ServeHTTP(w, r)
})
}
func InitLoggingResponseWriter(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
tn := time.Now()
writer := newLoggingResponseWriter(w)
h.ServeHTTP(writer, r)
ts := time.Since(tn)
log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode, "time", ts)
metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).Update(ts)
metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).Update(ts)
})
}
func InstrumentOpenTracing(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
uri := GetURI(r.Context())
if uri == nil || r.Method == "" || (uri != nil && uri.Scheme == "") {
h.ServeHTTP(w, r) // soft fail
return
}
spanName := fmt.Sprintf("http.%s.%s", r.Method, uri.Scheme)
ctx, sp := spancontext.StartSpan(r.Context(), spanName)
defer sp.Finish()
h.ServeHTTP(w, r.WithContext(ctx))
})
}
func RecoverPanic(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := recover(); err != nil {
log.Error("panic recovery!", "stack trace", string(debug.Stack()), "url", r.URL.String(), "headers", r.Header)
}
}()
h.ServeHTTP(w, r)
})
}

132
swarm/api/http/response.go
View file

@ -1,132 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"encoding/json"
"fmt"
"html/template"
"net/http"
"strings"
"time"
"github.com/ubiq/go-ubiq/log"
"github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/swarm/api"
)
var (
htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
plaintextCounter = metrics.NewRegisteredCounter("api.http.errorpage.plaintext.count", nil)
)
type ResponseParams struct {
Msg template.HTML
Code int
Timestamp string
template *template.Template
Details template.HTML
}
// ShowMultipleChoices is used when a user requests a resource in a manifest which results
// in ambiguous results. It returns a HTML page with clickable links of each of the entry
// in the manifest which fits the request URI ambiguity.
// For example, if the user requests bzz:/<hash>/read and that manifest contains entries
// "readme.md" and "readinglist.txt", a HTML page is returned with this two links.
// This only applies if the manifest has no default entry
func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) {
log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
msg := ""
if list.Entries == nil {
respondError(w, r, "Could not resolve", http.StatusInternalServerError)
return
}
requestUri := strings.TrimPrefix(r.RequestURI, "/")
uri, err := api.Parse(requestUri)
if err != nil {
respondError(w, r, "Bad Request", http.StatusBadRequest)
}
uri.Scheme = "bzz-list"
msg += fmt.Sprintf("Disambiguation:<br/>Your request may refer to multiple choices.<br/>Click <a class=\"orange\" href='"+"/"+uri.String()+"'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout(\"location.href='%s';\",5000);</script><br/>", "/"+uri.String())
respondTemplate(w, r, "error", msg, http.StatusMultipleChoices)
}
func respondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) {
log.Debug("respondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
respond(w, r, &ResponseParams{
Code: code,
Msg: template.HTML(msg),
Timestamp: time.Now().Format(time.RFC1123),
template: TemplatesMap[templateName],
})
}
func respondError(w http.ResponseWriter, r *http.Request, msg string, code int) {
log.Info("respondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code)
respondTemplate(w, r, "error", msg, code)
}
func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
w.WriteHeader(params.Code)
if params.Code >= 400 {
w.Header().Del("Cache-Control")
w.Header().Del("ETag")
}
acceptHeader := r.Header.Get("Accept")
// this cannot be in a switch since an Accept header can have multiple values: "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8"
if strings.Contains(acceptHeader, "application/json") {
if err := respondJSON(w, r, params); err != nil {
respondError(w, r, "Internal server error", http.StatusInternalServerError)
}
} else if strings.Contains(acceptHeader, "text/html") {
respondHTML(w, r, params)
} else {
respondPlaintext(w, r, params) //returns nice errors for curl
}
}
func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
htmlCounter.Inc(1)
log.Info("respondHTML", "ruid", GetRUID(r.Context()), "code", params.Code)
err := params.template.Execute(w, params)
if err != nil {
log.Error(err.Error())
}
}
func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
jsonCounter.Inc(1)
log.Info("respondJSON", "ruid", GetRUID(r.Context()), "code", params.Code)
w.Header().Set("Content-Type", "application/json")
return json.NewEncoder(w).Encode(params)
}
func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
plaintextCounter.Inc(1)
log.Info("respondPlaintext", "ruid", GetRUID(r.Context()), "code", params.Code)
w.Header().Set("Content-Type", "text/plain")
strToWrite := "Code: " + fmt.Sprintf("%d", params.Code) + "\n"
strToWrite += "Message: " + string(params.Msg) + "\n"
strToWrite += "Timestamp: " + params.Timestamp + "\n"
_, err := w.Write([]byte(strToWrite))
return err
}

170
swarm/api/http/response_test.go
View file

@ -1,170 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"encoding/json"
"io/ioutil"
"net/http"
"strings"
"testing"
"golang.org/x/net/html"
)
func TestError(t *testing.T) {
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
var respbody []byte
url := srv.URL + "/this_should_fail_as_no_bzz_protocol_present"
resp, err := http.Get(url)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
defer resp.Body.Close()
respbody, err = ioutil.ReadAll(resp.Body)
if resp.StatusCode != 404 && !strings.Contains(string(respbody), "Invalid URI &#34;/this_should_fail_as_no_bzz_protocol_present&#34;: unknown scheme") {
t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode)
}
_, err = html.Parse(strings.NewReader(string(respbody)))
if err != nil {
t.Fatalf("HTML validation failed for error page returned!")
}
}
func Test404Page(t *testing.T) {
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
var respbody []byte
url := srv.URL + "/bzz:/1234567890123456789012345678901234567890123456789012345678901234"
resp, err := http.Get(url)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
defer resp.Body.Close()
respbody, err = ioutil.ReadAll(resp.Body)
if resp.StatusCode != 404 || !strings.Contains(string(respbody), "404") {
t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
}
_, err = html.Parse(strings.NewReader(string(respbody)))
if err != nil {
t.Fatalf("HTML validation failed for error page returned!")
}
}
func Test500Page(t *testing.T) {
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
var respbody []byte
url := srv.URL + "/bzz:/thisShouldFailWith500Code"
resp, err := http.Get(url)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
defer resp.Body.Close()
respbody, err = ioutil.ReadAll(resp.Body)
if resp.StatusCode != 404 {
t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
}
_, err = html.Parse(strings.NewReader(string(respbody)))
if err != nil {
t.Fatalf("HTML validation failed for error page returned!")
}
}
func Test500PageWith0xHashPrefix(t *testing.T) {
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
var respbody []byte
url := srv.URL + "/bzz:/0xthisShouldFailWith500CodeAndAHelpfulMessage"
resp, err := http.Get(url)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
defer resp.Body.Close()
respbody, err = ioutil.ReadAll(resp.Body)
if resp.StatusCode != 404 {
t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
}
if !strings.Contains(string(respbody), "The requested hash seems to be prefixed with") {
t.Fatalf("Did not receive the expected error message")
}
_, err = html.Parse(strings.NewReader(string(respbody)))
if err != nil {
t.Fatalf("HTML validation failed for error page returned!")
}
}
func TestJsonResponse(t *testing.T) {
srv := NewTestSwarmServer(t, serverFunc, nil)
defer srv.Close()
var resp *http.Response
var respbody []byte
url := srv.URL + "/bzz:/thisShouldFailWith500Code/"
req, err := http.NewRequest("GET", url, nil)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
req.Header.Set("Accept", "application/json")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("Request failed: %v", err)
}
defer resp.Body.Close()
respbody, err = ioutil.ReadAll(resp.Body)
if resp.StatusCode != 404 {
t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode)
}
if !isJSON(string(respbody)) {
t.Fatalf("Expected response to be JSON, received invalid JSON: %s", string(respbody))
}
}
func isJSON(s string) bool {
var js map[string]interface{}
return json.Unmarshal([]byte(s), &js) == nil
}

66
swarm/api/http/roundtripper.go
View file

@ -1,66 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"fmt"
"net/http"
"github.com/ubiq/go-ubiq/swarm/log"
)
/*
http roundtripper to register for bzz url scheme
see https://github.com/ubiq/go-ubiq/issues/2040
Usage:
import (
"github.com/ubiq/go-ubiq/common/httpclient"
"github.com/ubiq/go-ubiq/swarm/api/http"
)
client := httpclient.New()
// for (private) swarm proxy running locally
client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
client.RegisterScheme("bzz-immutable", &http.RoundTripper{Port: port})
client.RegisterScheme("bzz-raw", &http.RoundTripper{Port: port})
The port you give the Roundtripper is the port the swarm proxy is listening on.
If Host is left empty, localhost is assumed.
Using a public gateway, the above few lines gives you the leanest
bzz-scheme aware read-only http client. You really only ever need this
if you need go-native swarm access to bzz addresses.
*/
type RoundTripper struct {
Host string
Port string
}
func (self *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) {
host := self.Host
if len(host) == 0 {
host = "localhost"
}
url := fmt.Sprintf("http://%s:%s/%s:/%s/%s", host, self.Port, req.Proto, req.URL.Host, req.URL.Path)
log.Info(fmt.Sprintf("roundtripper: proxying request '%s' to '%s'", req.RequestURI, url))
reqProxy, err := http.NewRequest(req.Method, url, req.Body)
if err != nil {
return nil, err
}
return http.DefaultClient.Do(reqProxy)
}

69
swarm/api/http/roundtripper_test.go
View file

@ -1,69 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestRoundTripper(t *testing.T) {
serveMux := http.NewServeMux()
serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" {
w.Header().Set("Content-Type", "text/plain")
http.ServeContent(w, r, "", time.Unix(0, 0), strings.NewReader(r.RequestURI))
} else {
http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed)
}
})
srv := httptest.NewServer(serveMux)
defer srv.Close()
host, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
rt := &RoundTripper{Host: host, Port: port}
trans := &http.Transport{}
trans.RegisterProtocol("bzz", rt)
client := &http.Client{Transport: trans}
resp, err := client.Get("bzz://test.com/path")
if err != nil {
t.Errorf("expected no error, got %v", err)
return
}
defer func() {
if resp != nil {
resp.Body.Close()
}
}()
content, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Errorf("expected no error, got %v", err)
return
}
if string(content) != "/HTTP/1.1:/test.com/path" {
t.Errorf("incorrect response from http server: expected '%v', got '%v'", "/HTTP/1.1:/test.com/path", string(content))
}
}

34
swarm/api/http/sctx.go
View file

@ -1,34 +0,0 @@
package http
import (
"context"
"github.com/ubiq/go-ubiq/swarm/api"
"github.com/ubiq/go-ubiq/swarm/sctx"
)
type uriKey struct{}
func GetRUID(ctx context.Context) string {
v, ok := ctx.Value(sctx.HTTPRequestIDKey{}).(string)
if ok {
return v
}
return "xxxxxxxx"
}
func SetRUID(ctx context.Context, ruid string) context.Context {
return context.WithValue(ctx, sctx.HTTPRequestIDKey{}, ruid)
}
func GetURI(ctx context.Context) *api.URI {
v, ok := ctx.Value(uriKey{}).(*api.URI)
if ok {
return v
}
return nil
}
func SetURI(ctx context.Context, uri *api.URI) context.Context {
return context.WithValue(ctx, uriKey{}, uri)
}

887
swarm/api/http/server.go
View file

@ -1,887 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
/*
A simple http server interface to Swarm
*/
package http
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/http"
"os"
"path"
"strconv"
"strings"
"time"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/metrics"
"github.com/ubiq/go-ubiq/swarm/api"
"github.com/ubiq/go-ubiq/swarm/log"
"github.com/ubiq/go-ubiq/swarm/storage"
"github.com/ubiq/go-ubiq/swarm/storage/feed"
"github.com/rs/cors"
)
var (
postRawCount = metrics.NewRegisteredCounter("api.http.post.raw.count", nil)
postRawFail = metrics.NewRegisteredCounter("api.http.post.raw.fail", nil)
postFilesCount = metrics.NewRegisteredCounter("api.http.post.files.count", nil)
postFilesFail = metrics.NewRegisteredCounter("api.http.post.files.fail", nil)
deleteCount = metrics.NewRegisteredCounter("api.http.delete.count", nil)
deleteFail = metrics.NewRegisteredCounter("api.http.delete.fail", nil)
getCount = metrics.NewRegisteredCounter("api.http.get.count", nil)
getFail = metrics.NewRegisteredCounter("api.http.get.fail", nil)
getFileCount = metrics.NewRegisteredCounter("api.http.get.file.count", nil)
getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil)
getFileFail = metrics.NewRegisteredCounter("api.http.get.file.fail", nil)
getListCount = metrics.NewRegisteredCounter("api.http.get.list.count", nil)
getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil)
)
type methodHandler map[string]http.Handler
func (m methodHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
v, ok := m[r.Method]
if ok {
v.ServeHTTP(rw, r)
return
}
rw.WriteHeader(http.StatusMethodNotAllowed)
}
func NewServer(api *api.API, corsString string) *Server {
var allowedOrigins []string
for _, domain := range strings.Split(corsString, ",") {
allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
}
c := cors.New(cors.Options{
AllowedOrigins: allowedOrigins,
AllowedMethods: []string{http.MethodPost, http.MethodGet, http.MethodDelete, http.MethodPatch, http.MethodPut},
MaxAge: 600,
AllowedHeaders: []string{"*"},
})
server := &Server{api: api}
defaultMiddlewares := []Adapter{
RecoverPanic,
SetRequestID,
SetRequestHost,
InitLoggingResponseWriter,
ParseURI,
InstrumentOpenTracing,
}
mux := http.NewServeMux()
mux.Handle("/bzz:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleBzzGet),
defaultMiddlewares...,
),
"POST": Adapt(
http.HandlerFunc(server.HandlePostFiles),
defaultMiddlewares...,
),
"DELETE": Adapt(
http.HandlerFunc(server.HandleDelete),
defaultMiddlewares...,
),
})
mux.Handle("/bzz-raw:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleGet),
defaultMiddlewares...,
),
"POST": Adapt(
http.HandlerFunc(server.HandlePostRaw),
defaultMiddlewares...,
),
})
mux.Handle("/bzz-immutable:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleBzzGet),
defaultMiddlewares...,
),
})
mux.Handle("/bzz-hash:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleGet),
defaultMiddlewares...,
),
})
mux.Handle("/bzz-list:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleGetList),
defaultMiddlewares...,
),
})
mux.Handle("/bzz-feed:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleGetFeed),
defaultMiddlewares...,
),
"POST": Adapt(
http.HandlerFunc(server.HandlePostFeed),
defaultMiddlewares...,
),
})
mux.Handle("/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleRootPaths),
SetRequestID,
InitLoggingResponseWriter,
),
})
server.Handler = c.Handler(mux)
return server
}
func (s *Server) ListenAndServe(addr string) error {
s.listenAddr = addr
return http.ListenAndServe(addr, s)
}
// browser API for registering bzz url scheme handlers:
// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
// electron (chromium) api for registering bzz url scheme handlers:
// https://github.com/atom/electron/blob/master/docs/api/protocol.md
type Server struct {
http.Handler
api *api.API
listenAddr string
}
func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) {
log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()), "uri", r.RequestURI)
if r.Header.Get("Accept") == "application/x-tar" {
uri := GetURI(r.Context())
_, credentials, _ := r.BasicAuth()
reader, err := s.api.GetDirectoryTar(r.Context(), s.api.Decryptor(r.Context(), credentials), uri)
if err != nil {
if isDecryptError(err) {
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", uri.Address().String()))
respondError(w, r, err.Error(), http.StatusUnauthorized)
return
}
respondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
return
}
defer reader.Close()
w.Header().Set("Content-Type", "application/x-tar")
fileName := uri.Addr
if found := path.Base(uri.Path); found != "" && found != "." && found != "/" {
fileName = found
}
w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=\"%s.tar\"", fileName))
w.WriteHeader(http.StatusOK)
io.Copy(w, reader)
return
}
s.HandleGetFile(w, r)
}
func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/":
respondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200)
return
case "/robots.txt":
w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
fmt.Fprintf(w, "User-agent: *\nDisallow: /")
case "/favicon.ico":
w.WriteHeader(http.StatusOK)
w.Write(faviconBytes)
default:
respondError(w, r, "Not Found", http.StatusNotFound)
}
}
// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
// body in swarm and returns the resulting storage address as a text/plain response
func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
log.Debug("handle.post.raw", "ruid", ruid)
postRawCount.Inc(1)
toEncrypt := false
uri := GetURI(r.Context())
if uri.Addr == "encrypt" {
toEncrypt = true
}
if uri.Path != "" {
postRawFail.Inc(1)
respondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest)
return
}
if uri.Addr != "" && uri.Addr != "encrypt" {
postRawFail.Inc(1)
respondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest)
return
}
if r.Header.Get("Content-Length") == "" {
postRawFail.Inc(1)
respondError(w, r, "missing Content-Length header in request", http.StatusBadRequest)
return
}
addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt)
if err != nil {
postRawFail.Inc(1)
respondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
log.Debug("stored content", "ruid", ruid, "key", addr)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, addr)
}
// HandlePostFiles handles a POST request to
// bzz:/<hash>/<path> which contains either a single file or multiple files
// (either a tar archive or multipart form), adds those files either to an
// existing manifest or to a new manifest under <path> and returns the
// resulting manifest hash as a text/plain response
func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
log.Debug("handle.post.files", "ruid", ruid)
postFilesCount.Inc(1)
contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil {
postFilesFail.Inc(1)
respondError(w, r, err.Error(), http.StatusBadRequest)
return
}
toEncrypt := false
uri := GetURI(r.Context())
if uri.Addr == "encrypt" {
toEncrypt = true
}
var addr storage.Address
if uri.Addr != "" && uri.Addr != "encrypt" {
addr, err = s.api.Resolve(r.Context(), uri.Addr)
if err != nil {
postFilesFail.Inc(1)
respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError)
return
}
log.Debug("resolved key", "ruid", ruid, "key", addr)
} else {
addr, err = s.api.NewManifest(r.Context(), toEncrypt)
if err != nil {
postFilesFail.Inc(1)
respondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
log.Debug("new manifest", "ruid", ruid, "key", addr)
}
newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error {
switch contentType {
case "application/x-tar":
_, err := s.handleTarUpload(r, mw)
if err != nil {
respondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)
return err
}
return nil
case "multipart/form-data":
return s.handleMultipartUpload(r, params["boundary"], mw)
default:
return s.handleDirectUpload(r, mw)
}
})
if err != nil {
postFilesFail.Inc(1)
respondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError)
return
}
log.Debug("stored content", "ruid", ruid, "key", newAddr)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, newAddr)
}
func (s *Server) handleTarUpload(r *http.Request, mw *api.ManifestWriter) (storage.Address, error) {
log.Debug("handle.tar.upload", "ruid", GetRUID(r.Context()))
defaultPath := r.URL.Query().Get("defaultpath")
key, err := s.api.UploadTar(r.Context(), r.Body, GetURI(r.Context()).Path, defaultPath, mw)
if err != nil {
return nil, err
}
return key, nil
}
func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api.ManifestWriter) error {
ruid := GetRUID(r.Context())
log.Debug("handle.multipart.upload", "ruid", ruid)
mr := multipart.NewReader(r.Body, boundary)
for {
part, err := mr.NextPart()
if err == io.EOF {
return nil
} else if err != nil {
return fmt.Errorf("error reading multipart form: %s", err)
}
var size int64
var reader io.Reader
if contentLength := part.Header.Get("Content-Length"); contentLength != "" {
size, err = strconv.ParseInt(contentLength, 10, 64)
if err != nil {
return fmt.Errorf("error parsing multipart content length: %s", err)
}
reader = part
} else {
// copy the part to a tmp file to get its size
tmp, err := ioutil.TempFile("", "swarm-multipart")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
defer tmp.Close()
size, err = io.Copy(tmp, part)
if err != nil {
return fmt.Errorf("error copying multipart content: %s", err)
}
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
return fmt.Errorf("error copying multipart content: %s", err)
}
reader = tmp
}
// add the entry under the path from the request
name := part.FileName()
if name == "" {
name = part.FormName()
}
uri := GetURI(r.Context())
path := path.Join(uri.Path, name)
entry := &api.ManifestEntry{
Path: path,
ContentType: part.Header.Get("Content-Type"),
Size: size,
}
log.Debug("adding path to new manifest", "ruid", ruid, "bytes", entry.Size, "path", entry.Path)
contentKey, err := mw.AddEntry(r.Context(), reader, entry)
if err != nil {
return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
}
log.Debug("stored content", "ruid", ruid, "key", contentKey)
}
}
func (s *Server) handleDirectUpload(r *http.Request, mw *api.ManifestWriter) error {
ruid := GetRUID(r.Context())
log.Debug("handle.direct.upload", "ruid", ruid)
key, err := mw.AddEntry(r.Context(), r.Body, &api.ManifestEntry{
Path: GetURI(r.Context()).Path,
ContentType: r.Header.Get("Content-Type"),
Mode: 0644,
Size: r.ContentLength,
})
if err != nil {
return err
}
log.Debug("stored content", "ruid", ruid, "key", key)
return nil
}
// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
// <path> from <manifest> and returns the resulting manifest hash as a
// text/plain response
func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.delete", "ruid", ruid)
deleteCount.Inc(1)
newKey, err := s.api.Delete(r.Context(), uri.Addr, uri.Path)
if err != nil {
deleteFail.Inc(1)
respondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, newKey)
}
// Handles feed manifest creation and feed updates
// The POST request admits a JSON structure as defined in the feeds package: `feed.updateRequestJSON`
// The requests can be to a) create a feed manifest, b) update a feed or c) both a+b: create a feed manifest and publish a first update
func (s *Server) HandlePostFeed(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.post.feed", "ruid", ruid)
var err error
// Creation and update must send feed.updateRequestJSON JSON structure
body, err := ioutil.ReadAll(r.Body)
if err != nil {
respondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
fd, err := s.api.ResolveFeed(r.Context(), uri, r.URL.Query())
if err != nil { // couldn't parse query string or retrieve manifest
getFail.Inc(1)
httpStatus := http.StatusBadRequest
if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI {
httpStatus = http.StatusNotFound
}
respondError(w, r, fmt.Sprintf("cannot retrieve feed from manifest: %s", err), httpStatus)
return
}
var updateRequest feed.Request
updateRequest.Feed = *fd
query := r.URL.Query()
if err := updateRequest.FromValues(query, body); err != nil { // decodes request from query parameters
respondError(w, r, err.Error(), http.StatusBadRequest)
return
}
switch {
case updateRequest.IsUpdate():
// Verify that the signature is intact and that the signer is authorized
// to update this feed
// Check this early, to avoid creating a feed and then not being able to set its first update.
if err = updateRequest.Verify(); err != nil {
respondError(w, r, err.Error(), http.StatusForbidden)
return
}
_, err = s.api.FeedsUpdate(r.Context(), &updateRequest)
if err != nil {
respondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
fallthrough
case query.Get("manifest") == "1":
// we create a manifest so we can retrieve feed updates with bzz:// later
// this manifest has a special "feed type" manifest, and saves the
// feed identification used to retrieve feed updates later
m, err := s.api.NewFeedManifest(r.Context(), &updateRequest.Feed)
if err != nil {
respondError(w, r, fmt.Sprintf("failed to create feed manifest: %v", err), http.StatusInternalServerError)
return
}
// the key to the manifest will be passed back to the client
// the client can access the feed directly through its Feed member
// the manifest key can be set as content in the resolver of the ENS name
outdata, err := json.Marshal(m)
if err != nil {
respondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError)
return
}
fmt.Fprint(w, string(outdata))
w.Header().Add("Content-type", "application/json")
default:
respondError(w, r, "Missing signature in feed update request", http.StatusBadRequest)
}
}
// HandleGetFeed retrieves Swarm feeds updates:
// bzz-feed://<manifest address or ENS name> - get latest feed update, given a manifest address
// - or -
// specify user + topic (optional), subtopic name (optional) directly, without manifest:
// bzz-feed://?user=0x...&topic=0x...&name=subtopic name
// topic defaults to 0x000... if not specified.
// name defaults to empty string if not specified.
// thus, empty name and topic refers to the user's default feed.
//
// Optional parameters:
// time=xx - get the latest update before time (in epoch seconds)
// hint.time=xx - hint the lookup algorithm looking for updates at around that time
// hint.level=xx - hint the lookup algorithm looking for updates at around this frequency level
// meta=1 - get feed metadata and status information instead of performing a feed query
// NOTE: meta=1 will be deprecated in the near future
func (s *Server) HandleGetFeed(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.get.feed", "ruid", ruid)
var err error
fd, err := s.api.ResolveFeed(r.Context(), uri, r.URL.Query())
if err != nil { // couldn't parse query string or retrieve manifest
getFail.Inc(1)
httpStatus := http.StatusBadRequest
if err == api.ErrCannotLoadFeedManifest || err == api.ErrCannotResolveFeedURI {
httpStatus = http.StatusNotFound
}
respondError(w, r, fmt.Sprintf("cannot retrieve feed information from manifest: %s", err), httpStatus)
return
}
// determine if the query specifies period and version or it is a metadata query
if r.URL.Query().Get("meta") == "1" {
unsignedUpdateRequest, err := s.api.FeedsNewRequest(r.Context(), fd)
if err != nil {
getFail.Inc(1)
respondError(w, r, fmt.Sprintf("cannot retrieve feed metadata for feed=%s: %s", fd.Hex(), err), http.StatusNotFound)
return
}
rawResponse, err := unsignedUpdateRequest.MarshalJSON()
if err != nil {
respondError(w, r, fmt.Sprintf("cannot encode unsigned feed update request: %v", err), http.StatusInternalServerError)
return
}
w.Header().Add("Content-type", "application/json")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, string(rawResponse))
return
}
lookupParams := &feed.Query{Feed: *fd}
if err = lookupParams.FromValues(r.URL.Query()); err != nil { // parse period, version
respondError(w, r, fmt.Sprintf("invalid feed update request:%s", err), http.StatusBadRequest)
return
}
data, err := s.api.FeedsLookup(r.Context(), lookupParams)
// any error from the switch statement will end up here
if err != nil {
code, err2 := s.translateFeedError(w, r, "feed lookup fail", err)
respondError(w, r, err2.Error(), code)
return
}
// All ok, serve the retrieved update
log.Debug("Found update", "feed", fd.Hex(), "ruid", ruid)
w.Header().Set("Content-Type", api.MimeOctetStream)
http.ServeContent(w, r, "", time.Now(), bytes.NewReader(data))
}
func (s *Server) translateFeedError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) {
code := 0
defaultErr := fmt.Errorf("%s: %v", supErr, err)
rsrcErr, ok := err.(*feed.Error)
if !ok && rsrcErr != nil {
code = rsrcErr.Code()
}
switch code {
case storage.ErrInvalidValue:
return http.StatusBadRequest, defaultErr
case storage.ErrNotFound, storage.ErrNotSynced, storage.ErrNothingToReturn, storage.ErrInit:
return http.StatusNotFound, defaultErr
case storage.ErrUnauthorized, storage.ErrInvalidSignature:
return http.StatusUnauthorized, defaultErr
case storage.ErrDataOverflow:
return http.StatusRequestEntityTooLarge, defaultErr
}
return http.StatusInternalServerError, defaultErr
}
// HandleGet handles a GET request to
// - bzz-raw://<key> and responds with the raw content stored at the
// given storage key
// - bzz-hash://<key> and responds with the hash of the content stored
// at the given storage key as a text/plain response
func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.get", "ruid", ruid, "uri", uri)
getCount.Inc(1)
_, pass, _ := r.BasicAuth()
addr, err := s.api.ResolveURI(r.Context(), uri, pass)
if err != nil {
getFail.Inc(1)
respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
log.Debug("handle.get: resolved", "ruid", ruid, "key", addr)
// if path is set, interpret <key> as a manifest and return the
// raw entry at the given path
etag := common.Bytes2Hex(addr)
noneMatchEtag := r.Header.Get("If-None-Match")
w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key.
if noneMatchEtag != "" {
if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), addr) {
w.WriteHeader(http.StatusNotModified)
return
}
}
// check the root chunk exists by retrieving the file's size
reader, isEncrypted := s.api.Retrieve(r.Context(), addr)
if _, err := reader.Size(r.Context(), nil); err != nil {
getFail.Inc(1)
respondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound)
return
}
w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted))
switch {
case uri.Raw():
// allow the request to overwrite the content type using a query
// parameter
if typ := r.URL.Query().Get("content_type"); typ != "" {
w.Header().Set("Content-Type", typ)
}
http.ServeContent(w, r, "", time.Now(), reader)
case uri.Hash():
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, addr)
}
}
// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
// a list of all files contained in <manifest> under <path> grouped into
// common prefixes using "/" as a delimiter
func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
_, credentials, _ := r.BasicAuth()
log.Debug("handle.get.list", "ruid", ruid, "uri", uri)
getListCount.Inc(1)
// ensure the root path has a trailing slash so that relative URLs work
if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
addr, err := s.api.Resolve(r.Context(), uri.Addr)
if err != nil {
getListFail.Inc(1)
respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr)
list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), addr, uri.Path)
if err != nil {
getListFail.Inc(1)
if isDecryptError(err) {
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", addr.String()))
respondError(w, r, err.Error(), http.StatusUnauthorized)
return
}
respondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
// if the client wants HTML (e.g. a browser) then render the list as a
// HTML index with relative URLs
if strings.Contains(r.Header.Get("Accept"), "text/html") {
w.Header().Set("Content-Type", "text/html")
err := TemplatesMap["bzz-list"].Execute(w, &htmlListData{
URI: &api.URI{
Scheme: "bzz",
Addr: uri.Addr,
Path: uri.Path,
},
List: &list,
})
if err != nil {
getListFail.Inc(1)
log.Error(fmt.Sprintf("error rendering list HTML: %s", err))
}
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(&list)
}
// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
// with the content of the file at <path> from the given <manifest>
func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
_, credentials, _ := r.BasicAuth()
log.Debug("handle.get.file", "ruid", ruid, "uri", r.RequestURI)
getFileCount.Inc(1)
// ensure the root path has a trailing slash so that relative URLs work
if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
var err error
manifestAddr := uri.Address()
if manifestAddr == nil {
manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr)
if err != nil {
getFileFail.Inc(1)
respondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
} else {
w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
}
log.Debug("handle.get.file: resolved", "ruid", ruid, "key", manifestAddr)
reader, contentType, status, contentKey, err := s.api.Get(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path)
etag := common.Bytes2Hex(contentKey)
noneMatchEtag := r.Header.Get("If-None-Match")
w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to actual content key.
if noneMatchEtag != "" {
if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), contentKey) {
w.WriteHeader(http.StatusNotModified)
return
}
}
if err != nil {
if isDecryptError(err) {
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr))
respondError(w, r, err.Error(), http.StatusUnauthorized)
return
}
switch status {
case http.StatusNotFound:
getFileNotFound.Inc(1)
respondError(w, r, err.Error(), http.StatusNotFound)
default:
getFileFail.Inc(1)
respondError(w, r, err.Error(), http.StatusInternalServerError)
}
return
}
//the request results in ambiguous files
//e.g. /read with readme.md and readinglist.txt available in manifest
if status == http.StatusMultipleChoices {
list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path)
if err != nil {
getFileFail.Inc(1)
if isDecryptError(err) {
w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr))
respondError(w, r, err.Error(), http.StatusUnauthorized)
return
}
respondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", ruid)
//show a nice page links to available entries
ShowMultipleChoices(w, r, list)
return
}
// check the root chunk exists by retrieving the file's size
if _, err := reader.Size(r.Context(), nil); err != nil {
getFileNotFound.Inc(1)
respondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound)
return
}
if contentType != "" {
w.Header().Set("Content-Type", contentType)
}
fileName := uri.Addr
if found := path.Base(uri.Path); found != "" && found != "." && found != "/" {
fileName = found
}
w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=\"%s\"", fileName))
http.ServeContent(w, r, fileName, time.Now(), newBufferedReadSeeker(reader, getFileBufferSize))
}
// The size of buffer used for bufio.Reader on LazyChunkReader passed to
// http.ServeContent in HandleGetFile.
// Warning: This value influences the number of chunk requests and chunker join goroutines
// per file request.
// Recommended value is 4 times the io.Copy default buffer value which is 32kB.
const getFileBufferSize = 4 * 32 * 1024
// bufferedReadSeeker wraps bufio.Reader to expose Seek method
// from the provied io.ReadSeeker in newBufferedReadSeeker.
type bufferedReadSeeker struct {
r io.Reader
s io.Seeker
}
// newBufferedReadSeeker creates a new instance of bufferedReadSeeker,
// out of io.ReadSeeker. Argument `size` is the size of the read buffer.
func newBufferedReadSeeker(readSeeker io.ReadSeeker, size int) bufferedReadSeeker {
return bufferedReadSeeker{
r: bufio.NewReaderSize(readSeeker, size),
s: readSeeker,
}
}
func (b bufferedReadSeeker) Read(p []byte) (n int, err error) {
return b.r.Read(p)
}
func (b bufferedReadSeeker) Seek(offset int64, whence int) (int64, error) {
return b.s.Seek(offset, whence)
}
type loggingResponseWriter struct {
http.ResponseWriter
statusCode int
}
func newLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
return &loggingResponseWriter{w, http.StatusOK}
}
func (lrw *loggingResponseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}
func isDecryptError(err error) bool {
return strings.Contains(err.Error(), api.ErrDecrypt.Error())
}

1313
swarm/api/http/server_test.go
View file

File diff suppressed because it is too large Load diff

306
swarm/api/http/templates.go
View file

File diff suppressed because one or more lines are too long

97
swarm/api/http/test_server.go
View file

@ -1,97 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
"github.com/ubiq/go-ubiq/swarm/api"
"github.com/ubiq/go-ubiq/swarm/storage"
"github.com/ubiq/go-ubiq/swarm/storage/feed"
)
type TestServer interface {
ServeHTTP(http.ResponseWriter, *http.Request)
}
func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, resolver api.Resolver) *TestSwarmServer {
dir, err := ioutil.TempDir("", "swarm-storage-test")
if err != nil {
t.Fatal(err)
}
storeparams := storage.NewDefaultLocalStoreParams()
storeparams.DbCapacity = 5000000
storeparams.CacheCapacity = 5000
storeparams.Init(dir)
localStore, err := storage.NewLocalStore(storeparams, nil)
if err != nil {
os.RemoveAll(dir)
t.Fatal(err)
}
fileStore := storage.NewFileStore(localStore, storage.NewFileStoreParams())
// Swarm feeds test setup
feedsDir, err := ioutil.TempDir("", "swarm-feeds-test")
if err != nil {
t.Fatal(err)
}
rhparams := &feed.HandlerParams{}
rh, err := feed.NewTestHandler(feedsDir, rhparams)
if err != nil {
t.Fatal(err)
}
a := api.NewAPI(fileStore, resolver, rh.Handler, nil)
srv := httptest.NewServer(serverFunc(a))
tss := &TestSwarmServer{
Server: srv,
FileStore: fileStore,
dir: dir,
Hasher: storage.MakeHashFunc(storage.DefaultHash)(),
cleanup: func() {
srv.Close()
rh.Close()
os.RemoveAll(dir)
os.RemoveAll(feedsDir)
},
CurrentTime: 42,
}
feed.TimestampProvider = tss
return tss
}
type TestSwarmServer struct {
*httptest.Server
Hasher storage.SwarmHash
FileStore *storage.FileStore
dir string
cleanup func()
CurrentTime uint64
}
func (t *TestSwarmServer) Close() {
t.cleanup()
}
func (t *TestSwarmServer) Now() feed.Timestamp {
return feed.Timestamp{Time: t.CurrentTime}
}

58
swarm/api/inspector.go
View file

@ -1,58 +0,0 @@
// Copyright 2019 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"context"
"github.com/ubiq/go-ubiq/swarm/network"
"github.com/ubiq/go-ubiq/swarm/storage"
)
type Inspector struct {
api *API
hive *network.Hive
netStore *storage.NetStore
}
func NewInspector(api *API, hive *network.Hive, netStore *storage.NetStore) *Inspector {
return &Inspector{api, hive, netStore}
}
// Hive prints the kademlia table
func (inspector *Inspector) Hive() string {
return inspector.hive.String()
}
type HasInfo struct {
Addr string `json:"address"`
Has bool `json:"has"`
}
// Has checks whether each chunk address is present in the underlying datastore,
// the bool in the returned structs indicates if the underlying datastore has
// the chunk stored with the given address (true), or not (false)
func (inspector *Inspector) Has(chunkAddresses []storage.Address) []HasInfo {
results := make([]HasInfo, 0)
for _, addr := range chunkAddresses {
res := HasInfo{}
res.Addr = addr.String()
res.Has = inspector.netStore.Has(context.Background(), addr)
results = append(results, res)
}
return results
}

585
swarm/api/manifest.go
View file

@ -1,585 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/ubiq/go-ubiq/swarm/storage/feed"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/swarm/log"
"github.com/ubiq/go-ubiq/swarm/storage"
)
const (
ManifestType = "application/bzz-manifest+json"
FeedContentType = "application/bzz-feed"
manifestSizeLimit = 5 * 1024 * 1024
)
// Manifest represents a swarm manifest
type Manifest struct {
Entries []ManifestEntry `json:"entries,omitempty"`
}
// ManifestEntry represents an entry in a swarm manifest
type ManifestEntry struct {
Hash string `json:"hash,omitempty"`
Path string `json:"path,omitempty"`
ContentType string `json:"contentType,omitempty"`
Mode int64 `json:"mode,omitempty"`
Size int64 `json:"size,omitempty"`
ModTime time.Time `json:"mod_time,omitempty"`
Status int `json:"status,omitempty"`
Access *AccessEntry `json:"access,omitempty"`
Feed *feed.Feed `json:"feed,omitempty"`
}
// ManifestList represents the result of listing files in a manifest
type ManifestList struct {
CommonPrefixes []string `json:"common_prefixes,omitempty"`
Entries []*ManifestEntry `json:"entries,omitempty"`
}
// NewManifest creates and stores a new, empty manifest
func (a *API) NewManifest(ctx context.Context, toEncrypt bool) (storage.Address, error) {
var manifest Manifest
data, err := json.Marshal(&manifest)
if err != nil {
return nil, err
}
addr, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), toEncrypt)
if err != nil {
return nil, err
}
err = wait(ctx)
return addr, err
}
// Manifest hack for supporting Swarm feeds from the bzz: scheme
// see swarm/api/api.go:API.Get() for more information
func (a *API) NewFeedManifest(ctx context.Context, feed *feed.Feed) (storage.Address, error) {
var manifest Manifest
entry := ManifestEntry{
Feed: feed,
ContentType: FeedContentType,
}
manifest.Entries = append(manifest.Entries, entry)
data, err := json.Marshal(&manifest)
if err != nil {
return nil, err
}
addr, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), false)
if err != nil {
return nil, err
}
err = wait(ctx)
return addr, err
}
// ManifestWriter is used to add and remove entries from an underlying manifest
type ManifestWriter struct {
api *API
trie *manifestTrie
quitC chan bool
}
func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWriter, error) {
trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt)
if err != nil {
return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
}
return &ManifestWriter{a, trie, quitC}, nil
}
// AddEntry stores the given data and adds the resulting address to the manifest
func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (addr storage.Address, err error) {
entry := newManifestTrieEntry(e, nil)
if data != nil {
var wait func(context.Context) error
addr, wait, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted)
if err != nil {
return nil, err
}
err = wait(ctx)
if err != nil {
return nil, err
}
entry.Hash = addr.Hex()
}
if entry.Hash == "" {
return addr, errors.New("missing entry hash")
}
m.trie.addEntry(entry, m.quitC)
return addr, nil
}
// RemoveEntry removes the given path from the manifest
func (m *ManifestWriter) RemoveEntry(path string) error {
m.trie.deleteEntry(path, m.quitC)
return nil
}
// Store stores the manifest, returning the resulting storage address
func (m *ManifestWriter) Store() (storage.Address, error) {
return m.trie.ref, m.trie.recalcAndStore()
}
// ManifestWalker is used to recursively walk the entries in the manifest and
// all of its submanifests
type ManifestWalker struct {
api *API
trie *manifestTrie
quitC chan bool
}
func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, decrypt DecryptFunc, quitC chan bool) (*ManifestWalker, error) {
trie, err := loadManifest(ctx, a.fileStore, addr, quitC, decrypt)
if err != nil {
return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
}
return &ManifestWalker{a, trie, quitC}, nil
}
// ErrSkipManifest is used as a return value from WalkFn to indicate that the
// manifest should be skipped
var ErrSkipManifest = errors.New("skip this manifest")
// WalkFn is the type of function called for each entry visited by a recursive
// manifest walk
type WalkFn func(entry *ManifestEntry) error
// Walk recursively walks the manifest calling walkFn for each entry in the
// manifest, including submanifests
func (m *ManifestWalker) Walk(walkFn WalkFn) error {
return m.walk(m.trie, "", walkFn)
}
func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
for _, entry := range &trie.entries {
if entry == nil {
continue
}
entry.Path = prefix + entry.Path
err := walkFn(&entry.ManifestEntry)
if err != nil {
if entry.ContentType == ManifestType && err == ErrSkipManifest {
continue
}
return err
}
if entry.ContentType != ManifestType {
continue
}
if err := trie.loadSubTrie(entry, nil); err != nil {
return err
}
if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil {
return err
}
}
return nil
}
type manifestTrie struct {
fileStore *storage.FileStore
entries [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry
ref storage.Address // if ref != nil, it is stored
encrypted bool
decrypt DecryptFunc
}
func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
return &manifestTrieEntry{
ManifestEntry: *entry,
subtrie: subtrie,
}
}
type manifestTrieEntry struct {
ManifestEntry
subtrie *manifestTrie
}
func loadManifest(ctx context.Context, fileStore *storage.FileStore, addr storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
log.Trace("manifest lookup", "addr", addr)
// retrieve manifest via FileStore
manifestReader, isEncrypted := fileStore.Retrieve(ctx, addr)
log.Trace("reader retrieved", "addr", addr)
return readManifest(manifestReader, addr, fileStore, isEncrypted, quitC, decrypt)
}
func readManifest(mr storage.LazySectionReader, addr storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
// TODO check size for oversized manifests
size, err := mr.Size(mr.Context(), quitC)
if err != nil { // size == 0
// can't determine size means we don't have the root chunk
log.Trace("manifest not found", "addr", addr)
err = fmt.Errorf("Manifest not Found")
return
}
if size > manifestSizeLimit {
log.Warn("manifest exceeds size limit", "addr", addr, "size", size, "limit", manifestSizeLimit)
err = fmt.Errorf("Manifest size of %v bytes exceeds the %v byte limit", size, manifestSizeLimit)
return
}
manifestData := make([]byte, size)
read, err := mr.Read(manifestData)
if int64(read) < size {
log.Trace("manifest not found", "addr", addr)
if err == nil {
err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size)
}
return
}
log.Debug("manifest retrieved", "addr", addr)
var man struct {
Entries []*manifestTrieEntry `json:"entries"`
}
err = json.Unmarshal(manifestData, &man)
if err != nil {
err = fmt.Errorf("Manifest %v is malformed: %v", addr.Log(), err)
log.Trace("malformed manifest", "addr", addr)
return
}
log.Trace("manifest entries", "addr", addr, "len", len(man.Entries))
trie = &manifestTrie{
fileStore: fileStore,
encrypted: isEncrypted,
decrypt: decrypt,
}
for _, entry := range man.Entries {
err = trie.addEntry(entry, quitC)
if err != nil {
return
}
}
return
}
func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) error {
mt.ref = nil // trie modified, hash needs to be re-calculated on demand
if entry.ManifestEntry.Access != nil {
if mt.decrypt == nil {
return errors.New("dont have decryptor")
}
err := mt.decrypt(&entry.ManifestEntry)
if err != nil {
return err
}
}
if len(entry.Path) == 0 {
mt.entries[256] = entry
return nil
}
b := entry.Path[0]
oldentry := mt.entries[b]
if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) {
mt.entries[b] = entry
return nil
}
cpl := 0
for (len(entry.Path) > cpl) && (len(oldentry.Path) > cpl) && (entry.Path[cpl] == oldentry.Path[cpl]) {
cpl++
}
if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
if mt.loadSubTrie(oldentry, quitC) != nil {
return nil
}
entry.Path = entry.Path[cpl:]
oldentry.subtrie.addEntry(entry, quitC)
oldentry.Hash = ""
return nil
}
commonPrefix := entry.Path[:cpl]
subtrie := &manifestTrie{
fileStore: mt.fileStore,
encrypted: mt.encrypted,
}
entry.Path = entry.Path[cpl:]
oldentry.Path = oldentry.Path[cpl:]
subtrie.addEntry(entry, quitC)
subtrie.addEntry(oldentry, quitC)
mt.entries[b] = newManifestTrieEntry(&ManifestEntry{
Path: commonPrefix,
ContentType: ManifestType,
}, subtrie)
return nil
}
func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
for _, e := range &mt.entries {
if e != nil {
cnt++
entry = e
}
}
return
}
func (mt *manifestTrie) deleteEntry(path string, quitC chan bool) {
mt.ref = nil // trie modified, hash needs to be re-calculated on demand
if len(path) == 0 {
mt.entries[256] = nil
return
}
b := path[0]
entry := mt.entries[b]
if entry == nil {
return
}
if entry.Path == path {
mt.entries[b] = nil
return
}
epl := len(entry.Path)
if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
if mt.loadSubTrie(entry, quitC) != nil {
return
}
entry.subtrie.deleteEntry(path[epl:], quitC)
entry.Hash = ""
// remove subtree if it has less than 2 elements
cnt, lastentry := entry.subtrie.getCountLast()
if cnt < 2 {
if lastentry != nil {
lastentry.Path = entry.Path + lastentry.Path
}
mt.entries[b] = lastentry
}
}
}
func (mt *manifestTrie) recalcAndStore() error {
if mt.ref != nil {
return nil
}
var buffer bytes.Buffer
buffer.WriteString(`{"entries":[`)
list := &Manifest{}
for _, entry := range &mt.entries {
if entry != nil {
if entry.Hash == "" { // TODO: paralellize
err := entry.subtrie.recalcAndStore()
if err != nil {
return err
}
entry.Hash = entry.subtrie.ref.Hex()
}
list.Entries = append(list.Entries, entry.ManifestEntry)
}
}
manifest, err := json.Marshal(list)
if err != nil {
return err
}
sr := bytes.NewReader(manifest)
ctx := context.TODO()
addr, wait, err2 := mt.fileStore.Store(ctx, sr, int64(len(manifest)), mt.encrypted)
if err2 != nil {
return err2
}
err2 = wait(ctx)
mt.ref = addr
return err2
}
func (mt *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
if entry.ManifestEntry.Access != nil {
if mt.decrypt == nil {
return errors.New("dont have decryptor")
}
err := mt.decrypt(&entry.ManifestEntry)
if err != nil {
return err
}
}
if entry.subtrie == nil {
hash := common.Hex2Bytes(entry.Hash)
entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC, mt.decrypt)
entry.Hash = "" // might not match, should be recalculated
}
return
}
func (mt *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error {
plen := len(prefix)
var start, stop int
if plen == 0 {
start = 0
stop = 256
} else {
start = int(prefix[0])
stop = start
}
for i := start; i <= stop; i++ {
select {
case <-quitC:
return fmt.Errorf("aborted")
default:
}
entry := mt.entries[i]
if entry != nil {
epl := len(entry.Path)
if entry.ContentType == ManifestType {
l := plen
if epl < l {
l = epl
}
if prefix[:l] == entry.Path[:l] {
err := mt.loadSubTrie(entry, quitC)
if err != nil {
return err
}
err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb)
if err != nil {
return err
}
}
} else {
if (epl >= plen) && (prefix == entry.Path[:plen]) {
cb(entry, rp+entry.Path[plen:])
}
}
}
}
return nil
}
func (mt *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
return mt.listWithPrefixInt(prefix, "", quitC, cb)
}
func (mt *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) {
log.Trace(fmt.Sprintf("findPrefixOf(%s)", path))
if len(path) == 0 {
return mt.entries[256], 0
}
//see if first char is in manifest entries
b := path[0]
entry = mt.entries[b]
if entry == nil {
return mt.entries[256], 0
}
epl := len(entry.Path)
log.Trace(fmt.Sprintf("path = %v entry.Path = %v epl = %v", path, entry.Path, epl))
if len(path) <= epl {
if entry.Path[:len(path)] == path {
if entry.ContentType == ManifestType {
err := mt.loadSubTrie(entry, quitC)
if err == nil && entry.subtrie != nil {
subentries := entry.subtrie.entries
for i := 0; i < len(subentries); i++ {
sub := subentries[i]
if sub != nil && sub.Path == "" {
return sub, len(path)
}
}
}
entry.Status = http.StatusMultipleChoices
}
pos = len(path)
return
}
return nil, 0
}
if path[:epl] == entry.Path {
log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
//the subentry is a manifest, load subtrie
if entry.ContentType == ManifestType && (strings.Contains(entry.Path, path) || strings.Contains(path, entry.Path)) {
err := mt.loadSubTrie(entry, quitC)
if err != nil {
return nil, 0
}
sub, pos := entry.subtrie.findPrefixOf(path[epl:], quitC)
if sub != nil {
entry = sub
pos += epl
return sub, pos
} else if path == entry.Path {
entry.Status = http.StatusMultipleChoices
}
} else {
//entry is not a manifest, return it
if path != entry.Path {
return nil, 0
}
}
}
return nil, 0
}
// file system manifest always contains regularized paths
// no leading or trailing slashes, only single slashes inside
func RegularSlashes(path string) (res string) {
for i := 0; i < len(path); i++ {
if (path[i] != '/') || ((i > 0) && (path[i-1] != '/')) {
res = res + path[i:i+1]
}
}
if (len(res) > 0) && (res[len(res)-1] == '/') {
res = res[:len(res)-1]
}
return
}
func (mt *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) {
path := RegularSlashes(spath)
var pos int
quitC := make(chan bool)
entry, pos = mt.findPrefixOf(path, quitC)
return entry, path[:pos]
}

175
swarm/api/manifest_test.go
View file

@ -1,175 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"testing"
"github.com/ubiq/go-ubiq/swarm/storage"
)
func manifest(paths ...string) (manifestReader storage.LazySectionReader) {
var entries []string
for _, path := range paths {
entry := fmt.Sprintf(`{"path":"%s"}`, path)
entries = append(entries, entry)
}
manifest := fmt.Sprintf(`{"entries":[%s]}`, strings.Join(entries, ","))
return &storage.LazyTestSectionReader{
SectionReader: io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest))),
}
}
func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...string) *manifestTrie {
quitC := make(chan bool)
fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
ref := make([]byte, fileStore.HashSize())
trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC, NOOPDecrypt)
if err != nil {
t.Errorf("unexpected error making manifest: %v", err)
}
checkEntry(t, path, match, multiple, trie)
return trie
}
func checkEntry(t *testing.T, path, match string, multiple bool, trie *manifestTrie) {
entry, fullpath := trie.getEntry(path)
if match == "-" && entry != nil {
t.Errorf("expected no match for '%s', got '%s'", path, fullpath)
} else if entry == nil {
if match != "-" {
t.Errorf("expected entry '%s' to match '%s', got no match", match, path)
}
} else if fullpath != match {
t.Errorf("incorrect entry retrieved for '%s'. expected path '%v', got '%s'", path, match, fullpath)
}
if multiple && entry.Status != http.StatusMultipleChoices {
t.Errorf("Expected %d Multiple Choices Status for path %s, match %s, got %d", http.StatusMultipleChoices, path, match, entry.Status)
} else if !multiple && entry != nil && entry.Status == http.StatusMultipleChoices {
t.Errorf("Were not expecting %d Multiple Choices Status for path %s, match %s, but got it", http.StatusMultipleChoices, path, match)
}
}
func TestGetEntry(t *testing.T) {
// file system manifest always contains regularized paths
testGetEntry(t, "a", "a", false, "a")
testGetEntry(t, "b", "-", false, "a")
testGetEntry(t, "/a//", "a", false, "a")
// fallback
testGetEntry(t, "/a", "", false, "")
testGetEntry(t, "/a/b", "a/b", false, "a/b")
// longest/deepest math
testGetEntry(t, "read", "read", true, "readme.md", "readit.md")
testGetEntry(t, "rf", "-", false, "readme.md", "readit.md")
testGetEntry(t, "readme", "readme", false, "readme.md")
testGetEntry(t, "readme", "-", false, "readit.md")
testGetEntry(t, "readme.md", "readme.md", false, "readme.md")
testGetEntry(t, "readme.md", "-", false, "readit.md")
testGetEntry(t, "readmeAmd", "-", false, "readit.md")
testGetEntry(t, "readme.mdffff", "-", false, "readme.md")
testGetEntry(t, "ab", "ab", true, "ab/cefg", "ab/cedh", "ab/kkkkkk")
testGetEntry(t, "ab/ce", "ab/ce", true, "ab/cefg", "ab/cedh", "ab/ceuuuuuuuuuu")
testGetEntry(t, "abc", "abc", true, "abcd", "abczzzzef", "abc/def", "abc/e/g")
testGetEntry(t, "a/b", "a/b", true, "a", "a/bc", "a/ba", "a/b/c")
testGetEntry(t, "a/b", "a/b", false, "a", "a/b", "a/bb", "a/b/c")
testGetEntry(t, "//a//b//", "a/b", false, "a", "a/b", "a/bb", "a/b/c")
}
func TestExactMatch(t *testing.T) {
quitC := make(chan bool)
mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map")
fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
ref := make([]byte, fileStore.HashSize())
trie, err := readManifest(mf, ref, fileStore, false, quitC, nil)
if err != nil {
t.Errorf("unexpected error making manifest: %v", err)
}
entry, _ := trie.getEntry("shouldBeExactMatch.css")
if entry.Path != "" {
t.Errorf("Expected entry to match %s, got: %s", "shouldBeExactMatch.css", entry.Path)
}
if entry.Status == http.StatusMultipleChoices {
t.Errorf("Got status %d, which is unexepcted", http.StatusMultipleChoices)
}
}
func TestDeleteEntry(t *testing.T) {
}
// TestAddFileWithManifestPath tests that adding an entry at a path which
// already exists as a manifest just adds the entry to the manifest rather
// than replacing the manifest with the entry
func TestAddFileWithManifestPath(t *testing.T) {
// create a manifest containing "ab" and "ac"
manifest, _ := json.Marshal(&Manifest{
Entries: []ManifestEntry{
{Path: "ab", Hash: "ab"},
{Path: "ac", Hash: "ac"},
},
})
reader := &storage.LazyTestSectionReader{
SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))),
}
fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams())
ref := make([]byte, fileStore.HashSize())
trie, err := readManifest(reader, ref, fileStore, false, nil, NOOPDecrypt)
if err != nil {
t.Fatal(err)
}
checkEntry(t, "ab", "ab", false, trie)
checkEntry(t, "ac", "ac", false, trie)
// now add path "a" and check we can still get "ab" and "ac"
entry := &manifestTrieEntry{}
entry.Path = "a"
entry.Hash = "a"
trie.addEntry(entry, nil)
checkEntry(t, "ab", "ab", false, trie)
checkEntry(t, "ac", "ac", false, trie)
checkEntry(t, "a", "a", false, trie)
}
// TestReadManifestOverSizeLimit creates a manifest reader with data longer then
// manifestSizeLimit and checks if readManifest function will return the exact error
// message.
// The manifest data is not in json-encoded format, preventing possbile
// successful parsing attempts if limit check fails.
func TestReadManifestOverSizeLimit(t *testing.T) {
manifest := make([]byte, manifestSizeLimit+1)
reader := &storage.LazyTestSectionReader{
SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))),
}
_, err := readManifest(reader, storage.Address{}, nil, false, nil, NOOPDecrypt)
if err == nil {
t.Fatal("got no error from readManifest")
}
// Error message is part of the http response body
// which justifies exact string validation.
got := err.Error()
want := fmt.Sprintf("Manifest size of %v bytes exceeds the %v byte limit", len(manifest), manifestSizeLimit)
if got != want {
t.Fatalf("got error mesage %q, expected %q", got, want)
}
}

85
swarm/api/storage.go
View file

@ -1,85 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"context"
"path"
"github.com/ubiq/go-ubiq/swarm/storage"
)
type Response struct {
MimeType string
Status int
Size int64
// Content []byte
Content string
}
// implements a service
//
// DEPRECATED: Use the HTTP API instead
type Storage struct {
api *API
}
func NewStorage(api *API) *Storage {
return &Storage{api}
}
// Put uploads the content to the swarm with a simple manifest speficying
// its content type
//
// DEPRECATED: Use the HTTP API instead
func (s *Storage) Put(ctx context.Context, content string, contentType string, toEncrypt bool) (storage.Address, func(context.Context) error, error) {
return s.api.Put(ctx, content, contentType, toEncrypt)
}
// Get retrieves the content from bzzpath and reads the response in full
// It returns the Response object, which serialises containing the
// response body as the value of the Content field
// NOTE: if error is non-nil, sResponse may still have partial content
// the actual size of which is given in len(resp.Content), while the expected
// size is resp.Size
//
// DEPRECATED: Use the HTTP API instead
func (s *Storage) Get(ctx context.Context, bzzpath string) (*Response, error) {
uri, err := Parse(path.Join("bzz:/", bzzpath))
if err != nil {
return nil, err
}
addr, err := s.api.Resolve(ctx, uri.Addr)
if err != nil {
return nil, err
}
reader, mimeType, status, _, err := s.api.Get(ctx, nil, addr, uri.Path)
if err != nil {
return nil, err
}
quitC := make(chan bool)
expsize, err := reader.Size(ctx, quitC)
if err != nil {
return nil, err
}
body := make([]byte, expsize)
size, err := reader.Read(body)
if int64(size) == expsize {
err = nil
}
return &Response{mimeType, status, expsize, string(body[:size])}, err
}

56
swarm/api/storage_test.go
View file

@ -1,56 +0,0 @@
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"context"
"testing"
)
func testStorage(t *testing.T, f func(*Storage, bool)) {
testAPI(t, func(api *API, toEncrypt bool) {
f(NewStorage(api), toEncrypt)
})
}
func TestStoragePutGet(t *testing.T) {
testStorage(t, func(api *Storage, toEncrypt bool) {
content := "hello"
exp := expResponse(content, "text/plain", 0)
// exp := expResponse([]byte(content), "text/plain", 0)
ctx := context.TODO()
bzzkey, wait, err := api.Put(ctx, content, exp.MimeType, toEncrypt)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
err = wait(ctx)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
bzzhash := bzzkey.Hex()
// to check put against the API#Get
resp0 := testGet(t, api.api, bzzhash, "")
checkResponse(t, resp0, exp)
// check storage#Get
resp, err := api.Get(context.TODO(), bzzhash)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
checkResponse(t, &testResponse{nil, resp}, exp)
})
}

BIN
swarm/api/testdata/test0/img/logo.png vendored
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 4 KiB

9
swarm/api/testdata/test0/index.css vendored
View file

@ -1,9 +0,0 @@
h1 {
color: black;
font-size: 12px;
background-color: orange;
border: 4px solid black;
}
body {
background-color: orange
}

10
swarm/api/testdata/test0/index.html vendored
View file

@ -1,10 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<link rel="stylesheet" href="index.css">
</head>
<body>
<h1>Swarm Test</h1>
<img src="img/logo.gif" align="center", alt="Ubiq logo">
</body>
</html>

144
swarm/api/uri.go
View file

@ -1,144 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"fmt"
"net/url"
"regexp"
"strings"
"github.com/ubiq/go-ubiq/common"
"github.com/ubiq/go-ubiq/swarm/storage"
)
//matches hex swarm hashes
// TODO: this is bad, it should not be hardcoded how long is a hash
var hashMatcher = regexp.MustCompile("^([0-9A-Fa-f]{64})([0-9A-Fa-f]{64})?$")
// URI is a reference to content stored in swarm.
type URI struct {
// Scheme has one of the following values:
//
// * bzz - an entry in a swarm manifest
// * bzz-raw - raw swarm content
// * bzz-immutable - immutable URI of an entry in a swarm manifest
// (address is not resolved)
// * bzz-list - list of all files contained in a swarm manifest
//
Scheme string
// Addr is either a hexadecimal storage address or it an address which
// resolves to a storage address
Addr string
// addr stores the parsed storage address
addr storage.Address
// Path is the path to the content within a swarm manifest
Path string
}
func (u *URI) MarshalJSON() (out []byte, err error) {
return []byte(`"` + u.String() + `"`), nil
}
func (u *URI) UnmarshalJSON(value []byte) error {
uri, err := Parse(string(value))
if err != nil {
return err
}
*u = *uri
return nil
}
// Parse parses rawuri into a URI struct, where rawuri is expected to have one
// of the following formats:
//
// * <scheme>:/
// * <scheme>:/<addr>
// * <scheme>:/<addr>/<path>
// * <scheme>://
// * <scheme>://<addr>
// * <scheme>://<addr>/<path>
//
// with scheme one of bzz, bzz-raw, bzz-immutable, bzz-list or bzz-hash
func Parse(rawuri string) (*URI, error) {
u, err := url.Parse(rawuri)
if err != nil {
return nil, err
}
uri := &URI{Scheme: u.Scheme}
// check the scheme is valid
switch uri.Scheme {
case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzz-hash", "bzz-feed":
default:
return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
}
// handle URIs like bzz://<addr>/<path> where the addr and path
// have already been split by url.Parse
if u.Host != "" {
uri.Addr = u.Host
uri.Path = strings.TrimLeft(u.Path, "/")
return uri, nil
}
// URI is like bzz:/<addr>/<path> so split the addr and path from
// the raw path (which will be /<addr>/<path>)
parts := strings.SplitN(strings.TrimLeft(u.Path, "/"), "/", 2)
uri.Addr = parts[0]
if len(parts) == 2 {
uri.Path = parts[1]
}
return uri, nil
}
func (u *URI) Feed() bool {
return u.Scheme == "bzz-feed"
}
func (u *URI) Raw() bool {
return u.Scheme == "bzz-raw"
}
func (u *URI) Immutable() bool {
return u.Scheme == "bzz-immutable"
}
func (u *URI) List() bool {
return u.Scheme == "bzz-list"
}
func (u *URI) Hash() bool {
return u.Scheme == "bzz-hash"
}
func (u *URI) String() string {
return u.Scheme + ":/" + u.Addr + "/" + u.Path
}
func (u *URI) Address() storage.Address {
if u.addr != nil {
return u.addr
}
if hashMatcher.MatchString(u.Addr) {
u.addr = common.Hex2Bytes(u.Addr)
return u.addr
}
return nil
}

175
swarm/api/uri_test.go
View file

@ -1,175 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"bytes"
"reflect"
"testing"
"github.com/ubiq/go-ubiq/swarm/storage"
)
func TestParseURI(t *testing.T) {
type test struct {
uri string
expectURI *URI
expectErr bool
expectRaw bool
expectImmutable bool
expectList bool
expectHash bool
expectValidKey bool
expectAddr storage.Address
}
tests := []test{
{
uri: "",
expectErr: true,
},
{
uri: "foo",
expectErr: true,
},
{
uri: "bzz",
expectErr: true,
},
{
uri: "bzz:",
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzz-immutable:",
expectURI: &URI{Scheme: "bzz-immutable"},
expectImmutable: true,
},
{
uri: "bzz-raw:",
expectURI: &URI{Scheme: "bzz-raw"},
expectRaw: true,
},
{
uri: "bzz:/",
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzz:/abc123",
expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
},
{
uri: "bzz:/abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
},
{
uri: "bzz-raw:/",
expectURI: &URI{Scheme: "bzz-raw"},
expectRaw: true,
},
{
uri: "bzz-raw:/abc123",
expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123"},
expectRaw: true,
},
{
uri: "bzz-raw:/abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123", Path: "path/to/entry"},
expectRaw: true,
},
{
uri: "bzz://",
expectURI: &URI{Scheme: "bzz"},
},
{
uri: "bzz://abc123",
expectURI: &URI{Scheme: "bzz", Addr: "abc123"},
},
{
uri: "bzz://abc123/path/to/entry",
expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
},
{
uri: "bzz-hash:",
expectURI: &URI{Scheme: "bzz-hash"},
expectHash: true,
},
{
uri: "bzz-hash:/",
expectURI: &URI{Scheme: "bzz-hash"},
expectHash: true,
},
{
uri: "bzz-list:",
expectURI: &URI{Scheme: "bzz-list"},
expectList: true,
},
{
uri: "bzz-list:/",
expectURI: &URI{Scheme: "bzz-list"},
expectList: true,
},
{
uri: "bzz-raw://4378d19c26590f1a818ed7d6a62c3809e149b0999cab5ce5f26233b3b423bf8c",
expectURI: &URI{Scheme: "bzz-raw",
Addr: "4378d19c26590f1a818ed7d6a62c3809e149b0999cab5ce5f26233b3b423bf8c",
},
expectValidKey: true,
expectRaw: true,
expectAddr: storage.Address{67, 120, 209, 156, 38, 89, 15, 26,
129, 142, 215, 214, 166, 44, 56, 9,
225, 73, 176, 153, 156, 171, 92, 229,
242, 98, 51, 179, 180, 35, 191, 140,
},
},
}
for _, x := range tests {
actual, err := Parse(x.uri)
if x.expectErr {
if err == nil {
t.Fatalf("expected %s to error", x.uri)
}
continue
}
if err != nil {
t.Fatalf("error parsing %s: %s", x.uri, err)
}
if !reflect.DeepEqual(actual, x.expectURI) {
t.Fatalf("expected %s to return %#v, got %#v", x.uri, x.expectURI, actual)
}
if actual.Raw() != x.expectRaw {
t.Fatalf("expected %s raw to be %t, got %t", x.uri, x.expectRaw, actual.Raw())
}
if actual.Immutable() != x.expectImmutable {
t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
}
if actual.List() != x.expectList {
t.Fatalf("expected %s list to be %t, got %t", x.uri, x.expectList, actual.List())
}
if actual.Hash() != x.expectHash {
t.Fatalf("expected %s hash to be %t, got %t", x.uri, x.expectHash, actual.Hash())
}
if x.expectValidKey {
if actual.Address() == nil {
t.Fatalf("expected %s to return a valid key, got nil", x.uri)
} else {
if !bytes.Equal(x.expectAddr, actual.Address()) {
t.Fatalf("expected %s to be decoded to %v", x.expectURI.Addr, x.expectAddr)
}
}
}
}
}

690
swarm/bmt/bmt.go
View file

@ -1,690 +0,0 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package bmt provides a binary merkle tree implementation used for swarm chunk hash
package bmt
import (
"fmt"
"hash"
"strings"
"sync"
"sync/atomic"
)
/*
Binary Merkle Tree Hash is a hash function over arbitrary datachunks of limited size.
It is defined as the root hash of the binary merkle tree built over fixed size segments
of the underlying chunk using any base hash function (e.g., keccak 256 SHA3).
Chunks with data shorter than the fixed size are hashed as if they had zero padding.
BMT hash is used as the chunk hash function in swarm which in turn is the basis for the
128 branching swarm hash http://swarm-guide.readthedocs.io/en/latest/architecture.html#swarm-hash
The BMT is optimal for providing compact inclusion proofs, i.e. prove that a
segment is a substring of a chunk starting at a particular offset.
The size of the underlying segments is fixed to the size of the base hash (called the resolution
of the BMT hash), Using Keccak256 SHA3 hash is 32 bytes, the EVM word size to optimize for on-chain BMT verification
as well as the hash size optimal for inclusion proofs in the merkle tree of the swarm hash.
Two implementations are provided:
* RefHasher is optimized for code simplicity and meant as a reference implementation
that is simple to understand
* Hasher is optimized for speed taking advantage of concurrency with minimalistic
control structure to coordinate the concurrent routines
BMT Hasher implements the following interfaces
* standard golang hash.Hash - synchronous, reusable
* SwarmHash - SumWithSpan provided
* io.Writer - synchronous left-to-right datawriter
* AsyncWriter - concurrent section writes and asynchronous Sum call
*/
const (
// PoolSize is the maximum number of bmt trees used by the hashers, i.e,
// the maximum number of concurrent BMT hashing operations performed by the same hasher
PoolSize = 8
)
// BaseHasherFunc is a hash.Hash constructor function used for the base hash of the BMT.
// implemented by Keccak256 SHA3 sha3.NewLegacyKeccak256
type BaseHasherFunc func() hash.Hash
// Hasher a reusable hasher for fixed maximum size chunks representing a BMT
// - implements the hash.Hash interface
// - reuses a pool of trees for amortised memory allocation and resource control
// - supports order-agnostic concurrent segment writes and section (double segment) writes
// as well as sequential read and write
// - the same hasher instance must not be called concurrently on more than one chunk
// - the same hasher instance is synchronously reuseable
// - Sum gives back the tree to the pool and guaranteed to leave
// the tree and itself in a state reusable for hashing a new chunk
// - generates and verifies segment inclusion proofs (TODO:)
type Hasher struct {
pool *TreePool // BMT resource pool
bmt *tree // prebuilt BMT resource for flowcontrol and proofs
}
// New creates a reusable BMT Hasher that
// pulls a new tree from a resource pool for hashing each chunk
func New(p *TreePool) *Hasher {
return &Hasher{
pool: p,
}
}
// TreePool provides a pool of trees used as resources by the BMT Hasher.
// A tree popped from the pool is guaranteed to have a clean state ready
// for hashing a new chunk.
type TreePool struct {
lock sync.Mutex
c chan *tree // the channel to obtain a resource from the pool
hasher BaseHasherFunc // base hasher to use for the BMT levels
SegmentSize int // size of leaf segments, stipulated to be = hash size
SegmentCount int // the number of segments on the base level of the BMT
Capacity int // pool capacity, controls concurrency
Depth int // depth of the bmt trees = int(log2(segmentCount))+1
Size int // the total length of the data (count * size)
count int // current count of (ever) allocated resources
zerohashes [][]byte // lookup table for predictable padding subtrees for all levels
}
// NewTreePool creates a tree pool with hasher, segment size, segment count and capacity
// on Hasher.getTree it reuses free trees or creates a new one if capacity is not reached
func NewTreePool(hasher BaseHasherFunc, segmentCount, capacity int) *TreePool {
// initialises the zerohashes lookup table
depth := calculateDepthFor(segmentCount)
segmentSize := hasher().Size()
zerohashes := make([][]byte, depth+1)
zeros := make([]byte, segmentSize)
zerohashes[0] = zeros
h := hasher()
for i := 1; i < depth+1; i++ {
zeros = doSum(h, nil, zeros, zeros)
zerohashes[i] = zeros
}
return &TreePool{
c: make(chan *tree, capacity),
hasher: hasher,
SegmentSize: segmentSize,
SegmentCount: segmentCount,
Capacity: capacity,
Size: segmentCount * segmentSize,
Depth: depth,
zerohashes: zerohashes,
}
}
// Drain drains the pool until it has no more than n resources
func (p *TreePool) Drain(n int) {
p.lock.Lock()
defer p.lock.Unlock()
for len(p.c) > n {
<-p.c
p.count--
}
}
// Reserve is blocking until it returns an available tree
// it reuses free trees or creates a new one if size is not reached
// TODO: should use a context here
func (p *TreePool) reserve() *tree {
p.lock.Lock()
defer p.lock.Unlock()
var t *tree
if p.count == p.Capacity {
return <-p.c
}
select {
case t = <-p.c:
default:
t = newTree(p.SegmentSize, p.Depth, p.hasher)
p.count++
}
return t
}
// release gives back a tree to the pool.
// this tree is guaranteed to be in reusable state
func (p *TreePool) release(t *tree) {
p.c <- t // can never fail ...
}
// tree is a reusable control structure representing a BMT
// organised in a binary tree
// Hasher uses a TreePool to obtain a tree for each chunk hash
// the tree is 'locked' while not in the pool
type tree struct {
leaves []*node // leaf nodes of the tree, other nodes accessible via parent links
cursor int // index of rightmost currently open segment
offset int // offset (cursor position) within currently open segment
section []byte // the rightmost open section (double segment)
result chan []byte // result channel
span []byte // The span of the data subsumed under the chunk
}
// node is a reuseable segment hasher representing a node in a BMT
type node struct {
isLeft bool // whether it is left side of the parent double segment
parent *node // pointer to parent node in the BMT
state int32 // atomic increment impl concurrent boolean toggle
left, right []byte // this is where the two children sections are written
hasher hash.Hash // preconstructed hasher on nodes
}
// newNode constructs a segment hasher node in the BMT (used by newTree)
func newNode(index int, parent *node, hasher hash.Hash) *node {
return &node{
parent: parent,
isLeft: index%2 == 0,
hasher: hasher,
}
}
// Draw draws the BMT (badly)
func (t *tree) draw(hash []byte) string {
var left, right []string
var anc []*node
for i, n := range t.leaves {
left = append(left, fmt.Sprintf("%v", hashstr(n.left)))
if i%2 == 0 {
anc = append(anc, n.parent)
}
right = append(right, fmt.Sprintf("%v", hashstr(n.right)))
}
anc = t.leaves
var hashes [][]string
for l := 0; len(anc) > 0; l++ {
var nodes []*node
hash := []string{""}
for i, n := range anc {
hash = append(hash, fmt.Sprintf("%v|%v", hashstr(n.left), hashstr(n.right)))
if i%2 == 0 && n.parent != nil {
nodes = append(nodes, n.parent)
}
}
hash = append(hash, "")
hashes = append(hashes, hash)
anc = nodes
}
hashes = append(hashes, []string{"", fmt.Sprintf("%v", hashstr(hash)), ""})
total := 60
del := " "
var rows []string
for i := len(hashes) - 1; i >= 0; i-- {
var textlen int
hash := hashes[i]
for _, s := range hash {
textlen += len(s)
}
if total < textlen {
total = textlen + len(hash)
}
delsize := (total - textlen) / (len(hash) - 1)
if delsize > len(del) {
delsize = len(del)
}
row := fmt.Sprintf("%v: %v", len(hashes)-i-1, strings.Join(hash, del[:delsize]))
rows = append(rows, row)
}
rows = append(rows, strings.Join(left, " "))
rows = append(rows, strings.Join(right, " "))
return strings.Join(rows, "\n") + "\n"
}
// newTree initialises a tree by building up the nodes of a BMT
// - segment size is stipulated to be the size of the hash
func newTree(segmentSize, depth int, hashfunc func() hash.Hash) *tree {
n := newNode(0, nil, hashfunc())
prevlevel := []*node{n}
// iterate over levels and creates 2^(depth-level) nodes
// the 0 level is on double segment sections so we start at depth - 2 since
count := 2
for level := depth - 2; level >= 0; level-- {
nodes := make([]*node, count)
for i := 0; i < count; i++ {
parent := prevlevel[i/2]
var hasher hash.Hash
if level == 0 {
hasher = hashfunc()
}
nodes[i] = newNode(i, parent, hasher)
}
prevlevel = nodes
count *= 2
}
// the datanode level is the nodes on the last level
return &tree{
leaves: prevlevel,
result: make(chan []byte),
section: make([]byte, 2*segmentSize),
}
}
// methods needed to implement hash.Hash
// Size returns the size
func (h *Hasher) Size() int {
return h.pool.SegmentSize
}
// BlockSize returns the block size
func (h *Hasher) BlockSize() int {
return 2 * h.pool.SegmentSize
}
// Sum returns the BMT root hash of the buffer
// using Sum presupposes sequential synchronous writes (io.Writer interface)
// hash.Hash interface Sum method appends the byte slice to the underlying
// data before it calculates and returns the hash of the chunk
// caller must make sure Sum is not called concurrently with Write, writeSection
func (h *Hasher) Sum(b []byte) (s []byte) {
t := h.getTree()
// write the last section with final flag set to true
go h.writeSection(t.cursor, t.section, true, true)
// wait for the result
s = <-t.result
span := t.span
// release the tree resource back to the pool
h.releaseTree()
// b + sha3(span + BMT(pure_chunk))
if len(span) == 0 {
return append(b, s...)
}
return doSum(h.pool.hasher(), b, span, s)
}
// methods needed to implement the SwarmHash and the io.Writer interfaces
// Write calls sequentially add to the buffer to be hashed,
// with every full segment calls writeSection in a go routine
func (h *Hasher) Write(b []byte) (int, error) {
l := len(b)
if l == 0 || l > h.pool.Size {
return 0, nil
}
t := h.getTree()
secsize := 2 * h.pool.SegmentSize
// calculate length of missing bit to complete current open section
smax := secsize - t.offset
// if at the beginning of chunk or middle of the section
if t.offset < secsize {
// fill up current segment from buffer
copy(t.section[t.offset:], b)
// if input buffer consumed and open section not complete, then
// advance offset and return
if smax == 0 {
smax = secsize
}
if l <= smax {
t.offset += l
return l, nil
}
} else {
// if end of a section
if t.cursor == h.pool.SegmentCount*2 {
return 0, nil
}
}
// read full sections and the last possibly partial section from the input buffer
for smax < l {
// section complete; push to tree asynchronously
go h.writeSection(t.cursor, t.section, true, false)
// reset section
t.section = make([]byte, secsize)
// copy from input buffer at smax to right half of section
copy(t.section, b[smax:])
// advance cursor
t.cursor++
// smax here represents successive offsets in the input buffer
smax += secsize
}
t.offset = l - smax + secsize
return l, nil
}
// Reset needs to be called before writing to the hasher
func (h *Hasher) Reset() {
h.releaseTree()
}
// methods needed to implement the SwarmHash interface
// ResetWithLength needs to be called before writing to the hasher
// the argument is supposed to be the byte slice binary representation of
// the length of the data subsumed under the hash, i.e., span
func (h *Hasher) ResetWithLength(span []byte) {
h.Reset()
h.getTree().span = span
}
// releaseTree gives back the Tree to the pool whereby it unlocks
// it resets tree, segment and index
func (h *Hasher) releaseTree() {
t := h.bmt
if t == nil {
return
}
h.bmt = nil
go func() {
t.cursor = 0
t.offset = 0
t.span = nil
t.section = make([]byte, h.pool.SegmentSize*2)
select {
case <-t.result:
default:
}
h.pool.release(t)
}()
}
// NewAsyncWriter extends Hasher with an interface for concurrent segment/section writes
func (h *Hasher) NewAsyncWriter(double bool) *AsyncHasher {
secsize := h.pool.SegmentSize
if double {
secsize *= 2
}
write := func(i int, section []byte, final bool) {
h.writeSection(i, section, double, final)
}
return &AsyncHasher{
Hasher: h,
double: double,
secsize: secsize,
write: write,
}
}
// SectionWriter is an asynchronous segment/section writer interface
type SectionWriter interface {
Reset() // standard init to be called before reuse
Write(index int, data []byte) // write into section of index
Sum(b []byte, length int, span []byte) []byte // returns the hash of the buffer
SectionSize() int // size of the async section unit to use
}
// AsyncHasher extends BMT Hasher with an asynchronous segment/section writer interface
// AsyncHasher is unsafe and does not check indexes and section data lengths
// it must be used with the right indexes and length and the right number of sections
//
// behaviour is undefined if
// * non-final sections are shorter or longer than secsize
// * if final section does not match length
// * write a section with index that is higher than length/secsize
// * set length in Sum call when length/secsize < maxsec
//
// * if Sum() is not called on a Hasher that is fully written
// a process will block, can be terminated with Reset
// * it will not leak processes if not all sections are written but it blocks
// and keeps the resource which can be released calling Reset()
type AsyncHasher struct {
*Hasher // extends the Hasher
mtx sync.Mutex // to lock the cursor access
double bool // whether to use double segments (call Hasher.writeSection)
secsize int // size of base section (size of hash or double)
write func(i int, section []byte, final bool)
}
// methods needed to implement AsyncWriter
// SectionSize returns the size of async section unit to use
func (sw *AsyncHasher) SectionSize() int {
return sw.secsize
}
// Write writes the i-th section of the BMT base
// this function can and is meant to be called concurrently
// it sets max segment threadsafely
func (sw *AsyncHasher) Write(i int, section []byte) {
sw.mtx.Lock()
defer sw.mtx.Unlock()
t := sw.getTree()
// cursor keeps track of the rightmost section written so far
// if index is lower than cursor then just write non-final section as is
if i < t.cursor {
// if index is not the rightmost, safe to write section
go sw.write(i, section, false)
return
}
// if there is a previous rightmost section safe to write section
if t.offset > 0 {
if i == t.cursor {
// i==cursor implies cursor was set by Hash call so we can write section as final one
// since it can be shorter, first we copy it to the padded buffer
t.section = make([]byte, sw.secsize)
copy(t.section, section)
go sw.write(i, t.section, true)
return
}
// the rightmost section just changed, so we write the previous one as non-final
go sw.write(t.cursor, t.section, false)
}
// set i as the index of the righmost section written so far
// set t.offset to cursor*secsize+1
t.cursor = i
t.offset = i*sw.secsize + 1
t.section = make([]byte, sw.secsize)
copy(t.section, section)
}
// Sum can be called any time once the length and the span is known
// potentially even before all segments have been written
// in such cases Sum will block until all segments are present and
// the hash for the length can be calculated.
//
// b: digest is appended to b
// length: known length of the input (unsafe; undefined if out of range)
// meta: metadata to hash together with BMT root for the final digest
// e.g., span for protection against existential forgery
func (sw *AsyncHasher) Sum(b []byte, length int, meta []byte) (s []byte) {
sw.mtx.Lock()
t := sw.getTree()
if length == 0 {
sw.mtx.Unlock()
s = sw.pool.zerohashes[sw.pool.Depth]
} else {
// for non-zero input the rightmost section is written to the tree asynchronously
// if the actual last section has been written (t.cursor == length/t.secsize)
maxsec := (length - 1) / sw.secsize
if t.offset > 0 {
go sw.write(t.cursor, t.section, maxsec == t.cursor)
}
// set cursor to maxsec so final section is written when it arrives
t.cursor = maxsec
t.offset = length
result := t.result
sw.mtx.Unlock()
// wait for the result or reset
s = <-result
}
// relesase the tree back to the pool
sw.releaseTree()
// if no meta is given just append digest to b
if len(meta) == 0 {
return append(b, s...)
}
// hash together meta and BMT root hash using the pools
return doSum(sw.pool.hasher(), b, meta, s)
}
// writeSection writes the hash of i-th section into level 1 node of the BMT tree
func (h *Hasher) writeSection(i int, section []byte, double bool, final bool) {
// select the leaf node for the section
var n *node
var isLeft bool
var hasher hash.Hash
var level int
t := h.getTree()
if double {
level++
n = t.leaves[i]
hasher = n.hasher
isLeft = n.isLeft
n = n.parent
// hash the section
section = doSum(hasher, nil, section)
} else {
n = t.leaves[i/2]
hasher = n.hasher
isLeft = i%2 == 0
}
// write hash into parent node
if final {
// for the last segment use writeFinalNode
h.writeFinalNode(level, n, hasher, isLeft, section)
} else {
h.writeNode(n, hasher, isLeft, section)
}
}
// writeNode pushes the data to the node
// if it is the first of 2 sisters written, the routine terminates
// if it is the second, it calculates the hash and writes it
// to the parent node recursively
// since hashing the parent is synchronous the same hasher can be used
func (h *Hasher) writeNode(n *node, bh hash.Hash, isLeft bool, s []byte) {
level := 1
for {
// at the root of the bmt just write the result to the result channel
if n == nil {
h.getTree().result <- s
return
}
// otherwise assign child hash to left or right segment
if isLeft {
n.left = s
} else {
n.right = s
}
// the child-thread first arriving will terminate
if n.toggle() {
return
}
// the thread coming second now can be sure both left and right children are written
// so it calculates the hash of left|right and pushes it to the parent
s = doSum(bh, nil, n.left, n.right)
isLeft = n.isLeft
n = n.parent
level++
}
}
// writeFinalNode is following the path starting from the final datasegment to the
// BMT root via parents
// for unbalanced trees it fills in the missing right sister nodes using
// the pool's lookup table for BMT subtree root hashes for all-zero sections
// otherwise behaves like `writeNode`
func (h *Hasher) writeFinalNode(level int, n *node, bh hash.Hash, isLeft bool, s []byte) {
for {
// at the root of the bmt just write the result to the result channel
if n == nil {
if s != nil {
h.getTree().result <- s
}
return
}
var noHash bool
if isLeft {
// coming from left sister branch
// when the final section's path is going via left child node
// we include an all-zero subtree hash for the right level and toggle the node.
n.right = h.pool.zerohashes[level]
if s != nil {
n.left = s
// if a left final node carries a hash, it must be the first (and only thread)
// so the toggle is already in passive state no need no call
// yet thread needs to carry on pushing hash to parent
noHash = false
} else {
// if again first thread then propagate nil and calculate no hash
noHash = n.toggle()
}
} else {
// right sister branch
if s != nil {
// if hash was pushed from right child node, write right segment change state
n.right = s
// if toggle is true, we arrived first so no hashing just push nil to parent
noHash = n.toggle()
} else {
// if s is nil, then thread arrived first at previous node and here there will be two,
// so no need to do anything and keep s = nil for parent
noHash = true
}
}
// the child-thread first arriving will just continue resetting s to nil
// the second thread now can be sure both left and right children are written
// it calculates the hash of left|right and pushes it to the parent
if noHash {
s = nil
} else {
s = doSum(bh, nil, n.left, n.right)
}
// iterate to parent
isLeft = n.isLeft
n = n.parent
level++
}
}
// getTree obtains a BMT resource by reserving one from the pool and assigns it to the bmt field
func (h *Hasher) getTree() *tree {
if h.bmt != nil {
return h.bmt
}
t := h.pool.reserve()
h.bmt = t
return t
}
// atomic bool toggle implementing a concurrent reusable 2-state object
// atomic addint with %2 implements atomic bool toggle
// it returns true if the toggler just put it in the active/waiting state
func (n *node) toggle() bool {
return atomic.AddInt32(&n.state, 1)%2 == 1
}
// calculates the hash of the data using hash.Hash
func doSum(h hash.Hash, b []byte, data ...[]byte) []byte {
h.Reset()
for _, v := range data {
h.Write(v)
}
return h.Sum(b)
}
// hashstr is a pretty printer for bytes used in tree.draw
func hashstr(b []byte) string {
end := len(b)
if end > 4 {
end = 4
}
return fmt.Sprintf("%x", b[:end])
}
// calculateDepthFor calculates the depth (number of levels) in the BMT tree
func calculateDepthFor(n int) (d int) {
c := 2
for ; c < n; c *= 2 {
d++
}
return d + 1
}

84
swarm/bmt/bmt_r.go
View file

@ -1,84 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package bmt is a simple nonconcurrent reference implementation for hashsize segment based
// Binary Merkle tree hash on arbitrary but fixed maximum chunksize
//
// This implementation does not take advantage of any paralellisms and uses
// far more memory than necessary, but it is easy to see that it is correct.
// It can be used for generating test cases for optimized implementations.
// There is extra check on reference hasher correctness in bmt_test.go
// * TestRefHasher
// * testBMTHasherCorrectness function
package bmt
import (
"hash"
)
// RefHasher is the non-optimized easy-to-read reference implementation of BMT
type RefHasher struct {
maxDataLength int // c * hashSize, where c = 2 ^ ceil(log2(count)), where count = ceil(length / hashSize)
sectionLength int // 2 * hashSize
hasher hash.Hash // base hash func (Keccak256 SHA3)
}
// NewRefHasher returns a new RefHasher
func NewRefHasher(hasher BaseHasherFunc, count int) *RefHasher {
h := hasher()
hashsize := h.Size()
c := 2
for ; c < count; c *= 2 {
}
return &RefHasher{
sectionLength: 2 * hashsize,
maxDataLength: c * hashsize,
hasher: h,
}
}
// Hash returns the BMT hash of the byte slice
// implements the SwarmHash interface
func (rh *RefHasher) Hash(data []byte) []byte {
// if data is shorter than the base length (maxDataLength), we provide padding with zeros
d := make([]byte, rh.maxDataLength)
length := len(data)
if length > rh.maxDataLength {
length = rh.maxDataLength
}
copy(d, data[:length])
return rh.hash(d, rh.maxDataLength)
}
// data has length maxDataLength = segmentSize * 2^k
// hash calls itself recursively on both halves of the given slice
// concatenates the results, and returns the hash of that
// if the length of d is 2 * segmentSize then just returns the hash of that section
func (rh *RefHasher) hash(data []byte, length int) []byte {
var section []byte
if length == rh.sectionLength {
// section contains two data segments (d)
section = data
} else {
// section contains hashes of left and right BMT subtreea
// to be calculated by calling hash recursively on left and right half of d
length /= 2
section = append(rh.hash(data[:length], length), rh.hash(data[length:], length)...)
}
rh.hasher.Reset()
rh.hasher.Write(section)
return rh.hasher.Sum(nil)
}

583
swarm/bmt/bmt_test.go
View file

@ -1,583 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bmt
import (
"bytes"
"encoding/binary"
"fmt"
"math/rand"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/ubiq/go-ubiq/swarm/testutil"
"golang.org/x/crypto/sha3"
)
// the actual data length generated (could be longer than max datalength of the BMT)
const BufferSize = 4128
const (
// segmentCount is the maximum number of segments of the underlying chunk
// Should be equal to max-chunk-data-size / hash-size
// Currently set to 128 == 4096 (default chunk size) / 32 (sha3.keccak256 size)
segmentCount = 128
)
var counts = []int{1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 32, 37, 42, 53, 63, 64, 65, 111, 127, 128}
// calculates the Keccak256 SHA3 hash of the data
func sha3hash(data ...[]byte) []byte {
h := sha3.NewLegacyKeccak256()
return doSum(h, nil, data...)
}
// TestRefHasher tests that the RefHasher computes the expected BMT hash for
// some small data lengths
func TestRefHasher(t *testing.T) {
// the test struct is used to specify the expected BMT hash for
// segment counts between from and to and lengths from 1 to datalength
type test struct {
from int
to int
expected func([]byte) []byte
}
var tests []*test
// all lengths in [0,64] should be:
//
// sha3hash(data)
//
tests = append(tests, &test{
from: 1,
to: 2,
expected: func(d []byte) []byte {
data := make([]byte, 64)
copy(data, d)
return sha3hash(data)
},
})
// all lengths in [3,4] should be:
//
// sha3hash(
// sha3hash(data[:64])
// sha3hash(data[64:])
// )
//
tests = append(tests, &test{
from: 3,
to: 4,
expected: func(d []byte) []byte {
data := make([]byte, 128)
copy(data, d)
return sha3hash(sha3hash(data[:64]), sha3hash(data[64:]))
},
})
// all segmentCounts in [5,8] should be:
//
// sha3hash(
// sha3hash(
// sha3hash(data[:64])
// sha3hash(data[64:128])
// )
// sha3hash(
// sha3hash(data[128:192])
// sha3hash(data[192:])
// )
// )
//
tests = append(tests, &test{
from: 5,
to: 8,
expected: func(d []byte) []byte {
data := make([]byte, 256)
copy(data, d)
return sha3hash(sha3hash(sha3hash(data[:64]), sha3hash(data[64:128])), sha3hash(sha3hash(data[128:192]), sha3hash(data[192:])))
},
})
// run the tests
for i, x := range tests {
for segmentCount := x.from; segmentCount <= x.to; segmentCount++ {
for length := 1; length <= segmentCount*32; length++ {
t.Run(fmt.Sprintf("%d_segments_%d_bytes", segmentCount, length), func(t *testing.T) {
data := testutil.RandomBytes(i, length)
expected := x.expected(data)
actual := NewRefHasher(sha3.NewLegacyKeccak256, segmentCount).Hash(data)
if !bytes.Equal(actual, expected) {
t.Fatalf("expected %x, got %x", expected, actual)
}
})
}
}
}
}
// tests if hasher responds with correct hash comparing the reference implementation return value
func TestHasherEmptyData(t *testing.T) {
hasher := sha3.NewLegacyKeccak256
var data []byte
for _, count := range counts {
t.Run(fmt.Sprintf("%d_segments", count), func(t *testing.T) {
pool := NewTreePool(hasher, count, PoolSize)
defer pool.Drain(0)
bmt := New(pool)
rbmt := NewRefHasher(hasher, count)
refHash := rbmt.Hash(data)
expHash := syncHash(bmt, nil, data)
if !bytes.Equal(expHash, refHash) {
t.Fatalf("hash mismatch with reference. expected %x, got %x", refHash, expHash)
}
})
}
}
// tests sequential write with entire max size written in one go
func TestSyncHasherCorrectness(t *testing.T) {
data := testutil.RandomBytes(1, BufferSize)
hasher := sha3.NewLegacyKeccak256
size := hasher().Size()
var err error
for _, count := range counts {
t.Run(fmt.Sprintf("segments_%v", count), func(t *testing.T) {
max := count * size
var incr int
capacity := 1
pool := NewTreePool(hasher, count, capacity)
defer pool.Drain(0)
for n := 0; n <= max; n += incr {
incr = 1 + rand.Intn(5)
bmt := New(pool)
err = testHasherCorrectness(bmt, hasher, data, n, count)
if err != nil {
t.Fatal(err)
}
}
})
}
}
// tests order-neutral concurrent writes with entire max size written in one go
func TestAsyncCorrectness(t *testing.T) {
data := testutil.RandomBytes(1, BufferSize)
hasher := sha3.NewLegacyKeccak256
size := hasher().Size()
whs := []whenHash{first, last, random}
for _, double := range []bool{false, true} {
for _, wh := range whs {
for _, count := range counts {
t.Run(fmt.Sprintf("double_%v_hash_when_%v_segments_%v", double, wh, count), func(t *testing.T) {
max := count * size
var incr int
capacity := 1
pool := NewTreePool(hasher, count, capacity)
defer pool.Drain(0)
for n := 1; n <= max; n += incr {
incr = 1 + rand.Intn(5)
bmt := New(pool)
d := data[:n]
rbmt := NewRefHasher(hasher, count)
exp := rbmt.Hash(d)
got := syncHash(bmt, nil, d)
if !bytes.Equal(got, exp) {
t.Fatalf("wrong sync hash for datalength %v: expected %x (ref), got %x", n, exp, got)
}
sw := bmt.NewAsyncWriter(double)
got = asyncHashRandom(sw, nil, d, wh)
if !bytes.Equal(got, exp) {
t.Fatalf("wrong async hash for datalength %v: expected %x, got %x", n, exp, got)
}
}
})
}
}
}
}
// Tests that the BMT hasher can be synchronously reused with poolsizes 1 and PoolSize
func TestHasherReuse(t *testing.T) {
t.Run(fmt.Sprintf("poolsize_%d", 1), func(t *testing.T) {
testHasherReuse(1, t)
})
t.Run(fmt.Sprintf("poolsize_%d", PoolSize), func(t *testing.T) {
testHasherReuse(PoolSize, t)
})
}
// tests if bmt reuse is not corrupting result
func testHasherReuse(poolsize int, t *testing.T) {
hasher := sha3.NewLegacyKeccak256
pool := NewTreePool(hasher, segmentCount, poolsize)
defer pool.Drain(0)
bmt := New(pool)
for i := 0; i < 100; i++ {
data := testutil.RandomBytes(1, BufferSize)
n := rand.Intn(bmt.Size())
err := testHasherCorrectness(bmt, hasher, data, n, segmentCount)
if err != nil {
t.Fatal(err)
}
}
}
// Tests if pool can be cleanly reused even in concurrent use by several hasher
func TestBMTConcurrentUse(t *testing.T) {
hasher := sha3.NewLegacyKeccak256
pool := NewTreePool(hasher, segmentCount, PoolSize)
defer pool.Drain(0)
cycles := 100
errc := make(chan error)
for i := 0; i < cycles; i++ {
go func() {
bmt := New(pool)
data := testutil.RandomBytes(1, BufferSize)
n := rand.Intn(bmt.Size())
errc <- testHasherCorrectness(bmt, hasher, data, n, 128)
}()
}
LOOP:
for {
select {
case <-time.NewTimer(5 * time.Second).C:
t.Fatal("timed out")
case err := <-errc:
if err != nil {
t.Fatal(err)
}
cycles--
if cycles == 0 {
break LOOP
}
}
}
}
// Tests BMT Hasher io.Writer interface is working correctly
// even multiple short random write buffers
func TestBMTWriterBuffers(t *testing.T) {
hasher := sha3.NewLegacyKeccak256
for _, count := range counts {
t.Run(fmt.Sprintf("%d_segments", count), func(t *testing.T) {
errc := make(chan error)
pool := NewTreePool(hasher, count, PoolSize)
defer pool.Drain(0)
n := count * 32
bmt := New(pool)
data := testutil.RandomBytes(1, n)
rbmt := NewRefHasher(hasher, count)
refHash := rbmt.Hash(data)
expHash := syncHash(bmt, nil, data)
if !bytes.Equal(expHash, refHash) {
t.Fatalf("hash mismatch with reference. expected %x, got %x", refHash, expHash)
}
attempts := 10
f := func() error {
bmt := New(pool)
bmt.Reset()
var buflen int
for offset := 0; offset < n; offset += buflen {
buflen = rand.Intn(n-offset) + 1
read, err := bmt.Write(data[offset : offset+buflen])
if err != nil {
return err
}
if read != buflen {
return fmt.Errorf("incorrect read. expected %v bytes, got %v", buflen, read)
}
}
hash := bmt.Sum(nil)
if !bytes.Equal(hash, expHash) {
return fmt.Errorf("hash mismatch. expected %x, got %x", hash, expHash)
}
return nil
}
for j := 0; j < attempts; j++ {
go func() {
errc <- f()
}()
}
timeout := time.NewTimer(2 * time.Second)
for {
select {
case err := <-errc:
if err != nil {
t.Fatal(err)
}
attempts--
if attempts == 0 {
return
}
case <-timeout.C:
t.Fatalf("timeout")
}
}
})
}
}
// helper function that compares reference and optimised implementations on
// correctness
func testHasherCorrectness(bmt *Hasher, hasher BaseHasherFunc, d []byte, n, count int) (err error) {
span := make([]byte, 8)
if len(d) < n {
n = len(d)
}
binary.BigEndian.PutUint64(span, uint64(n))
data := d[:n]
rbmt := NewRefHasher(hasher, count)
exp := sha3hash(span, rbmt.Hash(data))
got := syncHash(bmt, span, data)
if !bytes.Equal(got, exp) {
return fmt.Errorf("wrong hash: expected %x, got %x", exp, got)
}
return err
}
//
func BenchmarkBMT(t *testing.B) {
for size := 4096; size >= 128; size /= 2 {
t.Run(fmt.Sprintf("%v_size_%v", "SHA3", size), func(t *testing.B) {
benchmarkSHA3(t, size)
})
t.Run(fmt.Sprintf("%v_size_%v", "Baseline", size), func(t *testing.B) {
benchmarkBMTBaseline(t, size)
})
t.Run(fmt.Sprintf("%v_size_%v", "REF", size), func(t *testing.B) {
benchmarkRefHasher(t, size)
})
t.Run(fmt.Sprintf("%v_size_%v", "BMT", size), func(t *testing.B) {
benchmarkBMT(t, size)
})
}
}
type whenHash = int
const (
first whenHash = iota
last
random
)
func BenchmarkBMTAsync(t *testing.B) {
whs := []whenHash{first, last, random}
for size := 4096; size >= 128; size /= 2 {
for _, wh := range whs {
for _, double := range []bool{false, true} {
t.Run(fmt.Sprintf("double_%v_hash_when_%v_size_%v", double, wh, size), func(t *testing.B) {
benchmarkBMTAsync(t, size, wh, double)
})
}
}
}
}
func BenchmarkPool(t *testing.B) {
caps := []int{1, PoolSize}
for size := 4096; size >= 128; size /= 2 {
for _, c := range caps {
t.Run(fmt.Sprintf("poolsize_%v_size_%v", c, size), func(t *testing.B) {
benchmarkPool(t, c, size)
})
}
}
}
// benchmarks simple sha3 hash on chunks
func benchmarkSHA3(t *testing.B, n int) {
data := testutil.RandomBytes(1, n)
hasher := sha3.NewLegacyKeccak256
h := hasher()
t.ReportAllocs()
t.ResetTimer()
for i := 0; i < t.N; i++ {
doSum(h, nil, data)
}
}
// benchmarks the minimum hashing time for a balanced (for simplicity) BMT
// by doing count/segmentsize parallel hashings of 2*segmentsize bytes
// doing it on n PoolSize each reusing the base hasher
// the premise is that this is the minimum computation needed for a BMT
// therefore this serves as a theoretical optimum for concurrent implementations
func benchmarkBMTBaseline(t *testing.B, n int) {
hasher := sha3.NewLegacyKeccak256
hashSize := hasher().Size()
data := testutil.RandomBytes(1, hashSize)
t.ReportAllocs()
t.ResetTimer()
for i := 0; i < t.N; i++ {
count := int32((n-1)/hashSize + 1)
wg := sync.WaitGroup{}
wg.Add(PoolSize)
var i int32
for j := 0; j < PoolSize; j++ {
go func() {
defer wg.Done()
h := hasher()
for atomic.AddInt32(&i, 1) < count {
doSum(h, nil, data)
}
}()
}
wg.Wait()
}
}
// benchmarks BMT Hasher
func benchmarkBMT(t *testing.B, n int) {
data := testutil.RandomBytes(1, n)
hasher := sha3.NewLegacyKeccak256
pool := NewTreePool(hasher, segmentCount, PoolSize)
bmt := New(pool)
t.ReportAllocs()
t.ResetTimer()
for i := 0; i < t.N; i++ {
syncHash(bmt, nil, data)
}
}
// benchmarks BMT hasher with asynchronous concurrent segment/section writes
func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) {
data := testutil.RandomBytes(1, n)
hasher := sha3.NewLegacyKeccak256
pool := NewTreePool(hasher, segmentCount, PoolSize)
bmt := New(pool).NewAsyncWriter(double)
idxs, segments := splitAndShuffle(bmt.SectionSize(), data)
rand.Shuffle(len(idxs), func(i int, j int) {
idxs[i], idxs[j] = idxs[j], idxs[i]
})
t.ReportAllocs()
t.ResetTimer()
for i := 0; i < t.N; i++ {
asyncHash(bmt, nil, n, wh, idxs, segments)
}
}
// benchmarks 100 concurrent bmt hashes with pool capacity
func benchmarkPool(t *testing.B, poolsize, n int) {
data := testutil.RandomBytes(1, n)
hasher := sha3.NewLegacyKeccak256
pool := NewTreePool(hasher, segmentCount, poolsize)
cycles := 100
t.ReportAllocs()
t.ResetTimer()
wg := sync.WaitGroup{}
for i := 0; i < t.N; i++ {
wg.Add(cycles)
for j := 0; j < cycles; j++ {
go func() {
defer wg.Done()
bmt := New(pool)
syncHash(bmt, nil, data)
}()
}
wg.Wait()
}
}
// benchmarks the reference hasher
func benchmarkRefHasher(t *testing.B, n int) {
data := testutil.RandomBytes(1, n)
hasher := sha3.NewLegacyKeccak256
rbmt := NewRefHasher(hasher, 128)
t.ReportAllocs()
t.ResetTimer()
for i := 0; i < t.N; i++ {
rbmt.Hash(data)
}
}
// Hash hashes the data and the span using the bmt hasher
func syncHash(h *Hasher, span, data []byte) []byte {
h.ResetWithLength(span)
h.Write(data)
return h.Sum(nil)
}
func splitAndShuffle(secsize int, data []byte) (idxs []int, segments [][]byte) {
l := len(data)
n := l / secsize
if l%secsize > 0 {
n++
}
for i := 0; i < n; i++ {
idxs = append(idxs, i)
end := (i + 1) * secsize
if end > l {
end = l
}
section := data[i*secsize : end]
segments = append(segments, section)
}
rand.Shuffle(n, func(i int, j int) {
idxs[i], idxs[j] = idxs[j], idxs[i]
})
return idxs, segments
}
// splits the input data performs a random shuffle to mock async section writes
func asyncHashRandom(bmt SectionWriter, span []byte, data []byte, wh whenHash) (s []byte) {
idxs, segments := splitAndShuffle(bmt.SectionSize(), data)
return asyncHash(bmt, span, len(data), wh, idxs, segments)
}
// mock for async section writes for BMT SectionWriter
// requires a permutation (a random shuffle) of list of all indexes of segments
// and writes them in order to the appropriate section
// the Sum function is called according to the wh parameter (first, last, random [relative to segment writes])
func asyncHash(bmt SectionWriter, span []byte, l int, wh whenHash, idxs []int, segments [][]byte) (s []byte) {
bmt.Reset()
if l == 0 {
return bmt.Sum(nil, l, span)
}
c := make(chan []byte, 1)
hashf := func() {
c <- bmt.Sum(nil, l, span)
}
maxsize := len(idxs)
var r int
if wh == random {
r = rand.Intn(maxsize)
}
for i, idx := range idxs {
bmt.Write(idx, segments[idx])
if (wh == first || wh == random) && i == r {
go hashf()
}
}
if wh == last {
return bmt.Sum(nil, l, span)
}
return <-c
}

5
swarm/chunk/chunk.go
View file

@ -1,5 +0,0 @@
package chunk
const (
DefaultSize = 4096
)

2
swarm/dev/.dockerignore
View file

@ -1,2 +0,0 @@
bin/*
cluster/*

2
swarm/dev/.gitignore vendored
View file

@ -1,2 +0,0 @@
bin/*
cluster/*

42
swarm/dev/Dockerfile
View file

@ -1,42 +0,0 @@
FROM ubuntu:xenial
# install build + test dependencies
RUN apt-get update && \
apt-get install --yes --no-install-recommends \
ca-certificates \
curl \
fuse \
g++ \
gcc \
git \
iproute2 \
iputils-ping \
less \
libc6-dev \
make \
pkg-config \
&& \
apt-get clean
# install Go
ENV GO_VERSION 1.8.1
RUN curl -fSLo golang.tar.gz "https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz" && \
tar -xzf golang.tar.gz -C /usr/local && \
rm golang.tar.gz
ENV GOPATH /go
ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
# install docker CLI
RUN curl -fSLo docker.tar.gz https://get.docker.com/builds/Linux/x86_64/docker-17.04.0-ce.tgz && \
tar -xzf docker.tar.gz -C /usr/local/bin --strip-components=1 docker/docker && \
rm docker.tar.gz
# install jq
RUN curl -fSLo /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && \
chmod +x /usr/local/bin/jq
# install govendor
RUN go get -u github.com/kardianos/govendor
# add custom bashrc
ADD bashrc /root/.bashrc

14
swarm/dev/Makefile
View file

@ -1,14 +0,0 @@
.PHONY: build cluster test
default: build
build:
go build -o bin/swarm github.com/ubiq/go-ubiq/cmd/swarm
go build -o bin/gubiq github.com/ubiq/go-ubiq/cmd/gubiq
go build -o bin/bootnode github.com/ubiq/go-ubiq/cmd/bootnode
cluster: build
scripts/boot-cluster.sh
test:
go test -v github.com/ubiq/go-ubiq/swarm/...

20
swarm/dev/README.md
View file

@ -1,20 +0,0 @@
Swarm development environment
=============================
The Swarm development environment is a Linux bash shell which can be run in a
Docker container and provides a predictable build and test environment.
### Start the Docker container
Run the `run.sh` script to build the Docker image and run it, you will then be
at a bash prompt inside the `swarm/dev` directory.
### Build binaries
Run `make` to build the `swarm`, `gubiq` and `bootnode` binaries into the
`swarm/dev/bin` directory.
### Boot a cluster
Run `make cluster` to start a 3 node Swarm cluster, or run
`scripts/boot-cluster.sh --size N` to boot a cluster of size N.

21
swarm/dev/bashrc
View file

@ -1,21 +0,0 @@
export ROOT="${GOPATH}/src/github.com/ubiq/go-ubiq"
export PATH="${ROOT}/swarm/dev/bin:${PATH}"
cd "${ROOT}/swarm/dev"
cat <<WELCOME
=============================================
Welcome to the swarm development environment.
- Run 'make' to build the swarm, gubiq and bootnode binaries
- Run 'make test' to run the swarm unit tests
- Run 'make cluster' to start a swarm cluster
- Run 'exit' to exit the development environment
See the 'scripts' directory for some useful scripts.
=============================================
WELCOME

90
swarm/dev/run.sh
View file

@ -1,90 +0,0 @@
#!/usr/bin/env bash
#
# A script to build and run the Swarm development environment using Docker.
set -e
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
# DEFAULT_NAME is the default name for the Docker image and container
DEFAULT_NAME="swarm-dev"
usage() {
cat >&2 <<USAGE
usage: $0 [options]
Build and run the Swarm development environment.
Depends on Docker being installed locally.
OPTIONS:
-n, --name NAME Docker image and container name [default: ${DEFAULT_NAME}]
-d, --docker-args ARGS Custom args to pass to 'docker run' (e.g. '-p 8000:8000' to expose a port)
-h, --help Show this message
USAGE
}
main() {
local name="${DEFAULT_NAME}"
local docker_args=""
parse_args "$@"
build_image
run_image
}
parse_args() {
while true; do
case "$1" in
-h | --help)
usage
exit 0
;;
-n | --name)
if [[ -z "$2" ]]; then
echo "ERROR: --name flag requires an argument" >&2
exit 1
fi
name="$2"
shift 2
;;
-d | --docker-args)
if [[ -z "$2" ]]; then
echo "ERROR: --docker-args flag requires an argument" >&2
exit 1
fi
docker_args="$2"
shift 2
;;
*)
break
;;
esac
done
if [[ $# -ne 0 ]]; then
usage
echo "ERROR: invalid arguments" >&2
exit 1
fi
}
build_image() {
docker build --tag "${name}" "${ROOT}/swarm/dev"
}
run_image() {
exec docker run \
--privileged \
--interactive \
--tty \
--rm \
--hostname "${name}" \
--name "${name}" \
--volume "${ROOT}:/go/src/github.com/ubiq/go-ubiq" \
--volume "/var/run/docker.sock:/var/run/docker.sock" \
${docker_args} \
"${name}" \
/bin/bash
}
main "$@"

288
swarm/dev/scripts/boot-cluster.sh
View file

@ -1,288 +0,0 @@
#!/bin/bash
#
# A script to boot a dev swarm cluster on a Linux host (typically in a Docker
# container started with swarm/dev/run.sh).
#
# The cluster contains a bootnode, a gubiq node and multiple swarm nodes, with
# each node having its own data directory in a base directory passed with the
# --dir flag (default is swarm/dev/cluster).
#
# To avoid using different ports for each node and to make networking more
# realistic, each node gets its own network namespace with IPs assigned from
# the 192.168.33.0/24 subnet:
#
# bootnode: 192.168.33.2
# gubiq: 192.168.33.3
# swarm: 192.168.33.10{1,2,...,n}
set -e
ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
source "${ROOT}/swarm/dev/scripts/util.sh"
# DEFAULT_BASE_DIR is the default base directory to store node data
DEFAULT_BASE_DIR="${ROOT}/swarm/dev/cluster"
# DEFAULT_CLUSTER_SIZE is the default swarm cluster size
DEFAULT_CLUSTER_SIZE=3
# Linux bridge configuration for connecting the node network namespaces
BRIDGE_NAME="swarmbr0"
BRIDGE_IP="192.168.33.1"
# static bootnode configuration
BOOTNODE_IP="192.168.33.2"
BOOTNODE_PORT="30301"
BOOTNODE_KEY="32078f313bea771848db70745225c52c00981589ad6b5b49163f0f5ee852617d"
BOOTNODE_PUBKEY="760c4460e5336ac9bbd87952a3c7ec4363fc0a97bd31c86430806e287b437fd1b01abc6e1db640cf3106b520344af1d58b00b57823db3e1407cbc433e1b6d04d"
BOOTNODE_URL="enode://${BOOTNODE_PUBKEY}@${BOOTNODE_IP}:${BOOTNODE_PORT}"
# static gubiq configuration
GUBIQ_IP="192.168.33.3"
GUBIQ_RPC_PORT="8588"
GUBIQ_RPC_URL="http://${GUBIQ_IP}:${GUBIQ_RPC_PORT}"
usage() {
cat >&2 <<USAGE
usage: $0 [options]
Boot a dev swarm cluster.
OPTIONS:
-d, --dir DIR Base directory to store node data [default: ${DEFAULT_BASE_DIR}]
-s, --size SIZE Size of swarm cluster [default: ${DEFAULT_CLUSTER_SIZE}]
-h, --help Show this message
USAGE
}
main() {
local base_dir="${DEFAULT_BASE_DIR}"
local cluster_size="${DEFAULT_CLUSTER_SIZE}"
parse_args "$@"
local pid_dir="${base_dir}/pids"
local log_dir="${base_dir}/logs"
mkdir -p "${base_dir}" "${pid_dir}" "${log_dir}"
stop_cluster
create_network
start_bootnode
start_gubiq_node
start_swarm_nodes
}
parse_args() {
while true; do
case "$1" in
-h | --help)
usage
exit 0
;;
-d | --dir)
if [[ -z "$2" ]]; then
fail "--dir flag requires an argument"
fi
base_dir="$2"
shift 2
;;
-s | --size)
if [[ -z "$2" ]]; then
fail "--size flag requires an argument"
fi
cluster_size="$2"
shift 2
;;
*)
break
;;
esac
done
if [[ $# -ne 0 ]]; then
usage
fail "ERROR: invalid arguments: $@"
fi
}
stop_cluster() {
info "stopping existing cluster"
"${ROOT}/swarm/dev/scripts/stop-cluster.sh" --dir "${base_dir}"
}
# create_network creates a Linux bridge which is used to connect the node
# network namespaces together
create_network() {
local subnet="${BRIDGE_IP}/24"
info "creating ${subnet} network on ${BRIDGE_NAME}"
ip link add name "${BRIDGE_NAME}" type bridge
ip link set dev "${BRIDGE_NAME}" up
ip address add "${subnet}" dev "${BRIDGE_NAME}"
}
# start_bootnode starts a bootnode which is used to bootstrap the gubiq and
# swarm nodes
start_bootnode() {
local key_file="${base_dir}/bootnode.key"
echo -n "${BOOTNODE_KEY}" > "${key_file}"
local args=(
--addr "${BOOTNODE_IP}:${BOOTNODE_PORT}"
--nodekey "${key_file}"
--verbosity "6"
)
start_node "bootnode" "${BOOTNODE_IP}" "$(which bootnode)" ${args[@]}
}
# start_gubiq_node starts a gubiq node with --datadir pointing at <base-dir>/gubiq
# and a single, unlocked account with password "gubiq"
start_gubiq_node() {
local dir="${base_dir}/gubiq"
mkdir -p "${dir}"
local password="gubiq"
echo "${password}" > "${dir}/password"
# create an account if necessary
if [[ ! -e "${dir}/keystore" ]]; then
info "creating gubiq account"
create_account "${dir}" "${password}"
fi
# get the account address
local address="$(jq --raw-output '.address' ${dir}/keystore/*)"
if [[ -z "${address}" ]]; then
fail "failed to get gubiq account address"
fi
local args=(
--datadir "${dir}"
--networkid "321"
--bootnodes "${BOOTNODE_URL}"
--unlock "${address}"
--password "${dir}/password"
--rpc
--rpcaddr "${GUBIQ_IP}"
--rpcport "${GUBIQ_RPC_PORT}"
--verbosity "6"
)
start_node "gubiq" "${GUBIQ_IP}" "$(which gubiq)" ${args[@]}
}
start_swarm_nodes() {
for i in $(seq 1 ${cluster_size}); do
start_swarm_node "${i}"
done
}
# start_swarm_node starts a swarm node with a name like "swarmNN" (where NN is
# a zero-padded integer like "07"), --datadir pointing at <base-dir>/<name>
# (e.g. <base-dir>/swarm07) and a single account with <name> as the password
start_swarm_node() {
local num=$1
local name="swarm$(printf '%02d' ${num})"
local ip="192.168.33.1$(printf '%02d' ${num})"
local dir="${base_dir}/${name}"
mkdir -p "${dir}"
local password="${name}"
echo "${password}" > "${dir}/password"
# create an account if necessary
if [[ ! -e "${dir}/keystore" ]]; then
info "creating account for ${name}"
create_account "${dir}" "${password}"
fi
# get the account address
local address="$(jq --raw-output '.address' ${dir}/keystore/*)"
if [[ -z "${address}" ]]; then
fail "failed to get swarm account address"
fi
local args=(
--bootnodes "${BOOTNODE_URL}"
--datadir "${dir}"
--identity "${name}"
--ens-api "${GUBIQ_RPC_URL}"
--bzznetworkid "321"
--bzzaccount "${address}"
--password "${dir}/password"
--verbosity "6"
)
start_node "${name}" "${ip}" "$(which swarm)" ${args[@]}
}
# start_node runs the node command as a daemon in a network namespace
start_node() {
local name="$1"
local ip="$2"
local path="$3"
local cmd_args=${@:4}
info "starting ${name} with IP ${ip}"
create_node_network "${name}" "${ip}"
# add a marker to the log file
cat >> "${log_dir}/${name}.log" <<EOF
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
Starting ${name} node - $(date)
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
EOF
# run the command in the network namespace using start-stop-daemon to
# daemonise the process, sending all output to the log file
local daemon_args=(
--start
--background
--no-close
--make-pidfile
--pidfile "${pid_dir}/${name}.pid"
--exec "${path}"
)
if ! ip netns exec "${name}" start-stop-daemon ${daemon_args[@]} -- $cmd_args &>> "${log_dir}/${name}.log"; then
fail "could not start ${name}, check ${log_dir}/${name}.log"
fi
}
# create_node_network creates a network namespace and connects it to the Linux
# bridge using a veth pair
create_node_network() {
local name="$1"
local ip="$2"
# create the namespace
ip netns add "${name}"
# create the veth pair
local veth0="veth${name}0"
local veth1="veth${name}1"
ip link add name "${veth0}" type veth peer name "${veth1}"
# add one end to the bridge
ip link set dev "${veth0}" master "${BRIDGE_NAME}"
ip link set dev "${veth0}" up
# add the other end to the namespace, rename it eth0 and give it the ip
ip link set dev "${veth1}" netns "${name}"
ip netns exec "${name}" ip link set dev "${veth1}" name "eth0"
ip netns exec "${name}" ip link set dev "eth0" up
ip netns exec "${name}" ip address add "${ip}/24" dev "eth0"
}
create_account() {
local dir=$1
local password=$2
gubiq --datadir "${dir}" --password /dev/stdin account new <<< "${password}"
}
main "$@"

Some files were not shown because too many files have changed in this diff Show more