swarm/api: Exorcize multihash

swarm/api/api.go

@@ -42,7 +42,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/swarm/log"
-	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/spancontext"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed"
@@ -417,7 +416,7 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
 				return reader, mimeType, status, nil, err
 			}
 			// get the data of the update
-			_, rsrcData, err := a.feed.GetContent(entry.Feed)
+			_, contentAddr, err := a.feed.GetContent(entry.Feed)
 			if err != nil {
 				apiGetNotFound.Inc(1)
 				status = http.StatusNotFound
@@ -425,23 +424,23 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
 				return reader, mimeType, status, nil, err
 			}
 
-			// extract multihash
+			// extract content hash
-			decodedMultihash, err := multihash.FromMultihash(rsrcData)
+			if len(contentAddr) != storage.AddressLength {
-			if err != nil {
 				apiGetInvalid.Inc(1)
 				status = http.StatusUnprocessableEntity
-				log.Warn("invalid multihash in feed update", "err", err)
+				errorMessage := fmt.Sprintf("invalid swarm hash in feed update. Expected %d bytes. Got %d", storage.AddressLength, len(contentAddr))
-				return reader, mimeType, status, nil, err
+				log.Warn(errorMessage)
+				return reader, mimeType, status, nil, errors.New(errorMessage)
 			}
-			manifestAddr = storage.Address(decodedMultihash)
+			manifestAddr = storage.Address(contentAddr)
-			log.Trace("feed update contains multihash", "key", manifestAddr)
+			log.Trace("feed update contains swarm hash", "key", manifestAddr)
 
-			// get the manifest the multihash digest points to
+			// get the manifest the swarm hash points to
 			trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt)
 			if err != nil {
 				apiGetNotFound.Inc(1)
 				status = http.StatusNotFound
-				log.Warn(fmt.Sprintf("loadManifestTrie (feed update multihash) error: %v", err))
+				log.Warn(fmt.Sprintf("loadManifestTrie (feed update) error: %v", err))
 				return reader, mimeType, status, nil, err
 			}
 
@@ -451,8 +450,8 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
 			if entry == nil {
 				status = http.StatusNotFound
 				apiGetNotFound.Inc(1)
-				err = fmt.Errorf("manifest (feed update multihash) entry for '%s' not found", path)
+				err = fmt.Errorf("manifest (feed update) entry for '%s' not found", path)
-				log.Trace("manifest (feed update multihash) entry not found", "key", manifestAddr, "path", path)
+				log.Trace("manifest (feed update) entry not found", "key", manifestAddr, "path", path)
 				return reader, mimeType, status, nil, err
 			}
 		}

swarm/api/client/client_test.go

@@ -25,13 +25,13 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
-	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed"
 )
 
@@ -368,58 +368,99 @@ func newTestSigner() (*feed.GenericSigner, error) {
 	return feed.NewGenericSigner(privKey), nil
 }
 
-// test the transparent resolving of multihash feed updates with bzz:// scheme
+// Test the transparent resolving of feed updates with bzz:// scheme
 //
-// first upload data, and store the multihash to the resulting manifest in a feed update
+// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update.
-// retrieving the update with the multihash should return the manifest pointing directly to the data
+// This effectively uses a feed to store a pointer to content rather than the content itself
+// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data
 // and raw retrieve of that hash should return the data
-func TestClientCreateFeedMultihash(t *testing.T) {
+func TestClientBzzWithFeed(t *testing.T) {
 
 	signer, _ := newTestSigner()
 
+	// Initialize a Swarm test server
 	srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
-	client := NewClient(srv.URL)
+	swarmClient := NewClient(srv.URL)
 	defer srv.Close()
 
-	// add the data our multihash aliased manifest will point to
+	// put together some data for our test:
-	databytes := []byte("bar")
+	dataBytes := []byte(`
+	//
+	// Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update.
+	// So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it:
+	//
+	// MANIFEST HASH --> DATA
+	//
+	// Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this,
+	// we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash.
+	//
+	// FEED MANIFEST HASH --> MANIFEST HASH --> DATA
+	//
+	// Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash**
+	// stays constant, we have effectively created a fixed address to changing content. (Applause)
+	//
+	// FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2)
+	//
+	`)
 
-	swarmHash, err := client.UploadRaw(bytes.NewReader(databytes), int64(len(databytes)), false)
+	// Create a virtual File out of memory containing the above data
-	if err != nil {
+	f := &File{
-		t.Fatalf("Error uploading raw test data: %s", err)
+		ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)),
+		ManifestEntry: api.ManifestEntry{
+			ContentType: "text/plain",
+			Mode:        0660,
+			Size:        int64(len(dataBytes)),
+		},
 	}
 
-	s := common.FromHex(swarmHash)
+	// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
-	mh := multihash.ToMultihash(s)
+	manifestAddressHex, err := swarmClient.Upload(f, "", false)
+	if err != nil {
+		t.Fatalf("Error creating manifest: %s", err)
+	}
 
-	// our feed topic
+	// convert the hex-encoded manifest hash to a 32-byte slice
-	topic, _ := feed.NewTopic("foo.eth", nil)
+	manifestAddress := common.FromHex(manifestAddressHex)
 
-	createRequest := feed.NewFirstRequest(topic)
+	if len(manifestAddress) != storage.AddressLength {
+		t.Fatalf("Something went wrong. Got a hash of an unexpected length. Expected %d bytes. Got %d", storage.AddressLength, len(manifestAddress))
+	}
 
-	createRequest.SetData(mh)
+	// Now create a **feed manifest**. For that, we need a topic:
-	if err := createRequest.Sign(signer); err != nil {
+	topic, _ := feed.NewTopic("interesting topic indeed", nil)
+
+	// Build a feed request to update data
+	request := feed.NewFirstRequest(topic)
+
+	// Put the 32-byte address of the manifest into the feed update
+	request.SetData(manifestAddress)
+
+	// Sign the update
+	if err := request.Sign(signer); err != nil {
 		t.Fatalf("Error signing update: %s", err)
 	}
 
-	feedManifestHash, err := client.CreateFeedWithManifest(createRequest)
+	// Publish the update and at the same time request a **feed manifest** to be created
-
+	feedManifestAddressHex, err := swarmClient.CreateFeedWithManifest(request)
 	if err != nil {
 		t.Fatalf("Error creating feed manifest: %s", err)
 	}
 
-	correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b"
+	// Check we have received the exact **feed manifest** to be expected
-	if feedManifestHash != correctManifestAddrHex {
+	// given the topic and user signing the updates:
-		t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash)
+	correctFeedManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2"
+	if feedManifestAddressHex != correctFeedManifestAddrHex {
+		t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctFeedManifestAddrHex, feedManifestAddressHex)
 	}
 
 	// Check we get a not found error when trying to get feed updates with a made-up manifest
-	_, err = client.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
+	_, err = swarmClient.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
 	if err != ErrNoFeedUpdatesFound {
 		t.Fatalf("Expected to receive ErrNoFeedUpdatesFound error. Got: %s", err)
 	}
 
-	reader, err := client.QueryFeed(nil, correctManifestAddrHex)
+	// If we query the feed directly we should get **manifest hash** back:
+	reader, err := swarmClient.QueryFeed(nil, correctFeedManifestAddrHex)
 	if err != nil {
 		t.Fatalf("Error retrieving feed updates: %s", err)
 	}
@@ -428,10 +469,27 @@ func TestClientCreateFeedMultihash(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(mh, gotData) {
+
-		t.Fatalf("Expected: %v, got %v", mh, gotData)
+	//Check that indeed the **manifest hash** is retrieved
+	if !bytes.Equal(manifestAddress, gotData) {
+		t.Fatalf("Expected: %v, got %v", manifestAddress, gotData)
 	}
 
+	// Now the final test we were looking for: Use bzz://<feed-manifest> and that should resolve all manifests
+	// and return the original data directly:
+	f, err = swarmClient.Download(feedManifestAddressHex, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	gotData, err = ioutil.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check that we get back the original data:
+	if !bytes.Equal(dataBytes, gotData) {
+		t.Fatalf("Expected: %v, got %v", manifestAddress, gotData)
+	}
 }
 
 // TestClientCreateUpdateFeed will check that feeds can be created and updated via the HTTP client.

swarm/api/http/server_test.go

@@ -45,7 +45,6 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
-	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/swarm/storage/feed"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
@@ -69,60 +68,91 @@ func newTestSigner() (*feed.GenericSigner, error) {
 	return feed.NewGenericSigner(privKey), nil
 }
 
-// test the transparent resolving of multihash-containing feed updates with bzz:// scheme
+// Test the transparent resolving of feed updates with bzz:// scheme
 //
-// first upload data, and store the multihash to the resulting manifest in a feed update
+// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update.
-// retrieving the update with the multihash should return the manifest pointing directly to the data
+// This effectively uses a feed to store a pointer to content rather than the content itself
+// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data
 // and raw retrieve of that hash should return the data
-func TestBzzFeedMultihash(t *testing.T) {
+func TestBzzWithFeed(t *testing.T) {
 
 	signer, _ := newTestSigner()
 
+	// Initialize Swarm test server
 	srv := NewTestSwarmServer(t, serverFunc, nil)
 	defer srv.Close()
 
-	// add the data our multihash aliased manifest will point to
+	// put together some data for our test:
-	databytes := "bar"
+	dataBytes := []byte(`
-	testBzzUrl := fmt.Sprintf("%s/bzz:/", srv.URL)
+	//
-	resp, err := http.Post(testBzzUrl, "text/plain", bytes.NewReader([]byte(databytes)))
+	// Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update.
+	// So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it:
+	//
+	// MANIFEST HASH --> DATA
+	//
+	// Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this,
+	// we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash.
+	//
+	// FEED MANIFEST HASH --> MANIFEST HASH --> DATA
+	//
+	// Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash**
+	// stays constant, we have effectively created a fixed address to changing content. (Applause)
+	//
+	// FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2) ...
+	//
+	`)
+
+	// POST data to bzz and get back a content-addressed **manifest hash** pointing to it.
+	resp, err := http.Post(fmt.Sprintf("%s/bzz:/", srv.URL), "text/plain", bytes.NewReader([]byte(dataBytes)))
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
 		t.Fatalf("err %s", resp.Status)
 	}
-	b, err := ioutil.ReadAll(resp.Body)
+	manifestAddressHex, err := ioutil.ReadAll(resp.Body)
 
 	if err != nil {
 		t.Fatal(err)
 	}
-	s := common.FromHex(string(b))
-	mh := multihash.ToMultihash(s)
 
-	log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
+	manifestAddress := common.FromHex(string(manifestAddressHex))
 
-	topic, _ := feed.NewTopic("foo.eth", nil)
+	log.Info("added data", "manifest", string(manifestAddressHex))
+
+	// At this point we have uploaded the data and have a manifest pointing to it
+	// Now store that manifest address in a feed update.
+	// We also want a feed manifest, so we can use it to refer to the feed.
+
+	// First, create a topic for our feed:
+	topic, _ := feed.NewTopic("interesting topic indeed", nil)
+
+	// Create a feed update request:
 	updateRequest := feed.NewFirstRequest(topic)
 
-	updateRequest.SetData(mh)
+	// Store the **manifest address** as data into the feed update.
+	updateRequest.SetData(manifestAddress)
 
+	// Sign the update
 	if err := updateRequest.Sign(signer); err != nil {
 		t.Fatal(err)
 	}
-	log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
+	log.Info("added data", "data", common.ToHex(manifestAddress))
 
-	testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL))
+	// Build the feed update http request:
+	feedUpdateURL, err := url.Parse(fmt.Sprintf("%s/bzz-feed:/", srv.URL))
 	if err != nil {
 		t.Fatal(err)
 	}
-	query := testUrl.Query()
+	query := feedUpdateURL.Query()
 	body := updateRequest.AppendValues(query) // this adds all query parameters and returns the data to be posted
-	query.Set("manifest", "1")                // indicate we want a manifest back
+	query.Set("manifest", "1")                // indicate we want a feed manifest back
-	testUrl.RawQuery = query.Encode()
+	feedUpdateURL.RawQuery = query.Encode()
 
-	// create the multihash update
+	// submit the feed update request to Swarm
-	resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
+	resp, err = http.Post(feedUpdateURL.String(), "application/octet-stream", bytes.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -130,24 +160,25 @@ func TestBzzFeedMultihash(t *testing.T) {
 	if resp.StatusCode != http.StatusOK {
 		t.Fatalf("err %s", resp.Status)
 	}
-	b, err = ioutil.ReadAll(resp.Body)
+
+	feedManifestAddressHex, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatal(err)
 	}
-	rsrcResp := &storage.Address{}
+	feedManifestAddress := &storage.Address{}
-	err = json.Unmarshal(b, rsrcResp)
+	err = json.Unmarshal(feedManifestAddressHex, feedManifestAddress)
 	if err != nil {
-		t.Fatalf("data %s could not be unmarshaled: %v", b, err)
+		t.Fatalf("data %s could not be unmarshaled: %v", feedManifestAddressHex, err)
 	}
 
-	correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b"
+	correctManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2"
-	if rsrcResp.Hex() != correctManifestAddrHex {
+	if feedManifestAddress.Hex() != correctManifestAddrHex {
-		t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
+		t.Fatalf("Response feed manifest address mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestAddress.Hex())
 	}
 
 	// get bzz manifest transparent feed update resolve
-	testBzzUrl = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
+	getBzzURL := fmt.Sprintf("%s/bzz:/%s", srv.URL, feedManifestAddress)
-	resp, err = http.Get(testBzzUrl)
+	resp, err = http.Get(getBzzURL)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -155,12 +186,12 @@ func TestBzzFeedMultihash(t *testing.T) {
 	if resp.StatusCode != http.StatusOK {
 		t.Fatalf("err %s", resp.Status)
 	}
-	b, err = ioutil.ReadAll(resp.Body)
+	retrievedData, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(b, []byte(databytes)) {
+	if !bytes.Equal(retrievedData, []byte(dataBytes)) {
-		t.Fatalf("retrieved data mismatch, expected %x, got %x", databytes, b)
+		t.Fatalf("retrieved data mismatch, expected %x, got %x", dataBytes, retrievedData)
 	}
 }
 
@@ -245,7 +276,8 @@ func TestBzzFeed(t *testing.T) {
 		t.Fatalf("Expected manifest Feed '%s', got '%s'", correctFeedHex, manifest.Entries[0].Feed.Hex())
 	}
 
-	// get bzz manifest transparent feed update resolve
+	// take the chance to have bzz: crash on resolving a feed update that does not contain
+	// a swarm hash:
 	testBzzUrl := fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
 	resp, err = http.Get(testBzzUrl)
 	if err != nil {
@@ -253,7 +285,7 @@ func TestBzzFeed(t *testing.T) {
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode == http.StatusOK {
-		t.Fatal("Expected error status since feed update does not contain multihash. Received 200 OK")
+		t.Fatal("Expected error status since feed update does not contain a Swarm hash. Received 200 OK")
 	}
 	_, err = ioutil.ReadAll(resp.Body)
 	if err != nil {