Merge branch 'master' into 111_pow

accounts/abi/event.go

@@ -71,14 +71,16 @@ func (e Event) tupleUnpack(v interface{}, output []byte) error {
 		if input.Indexed {
 			// can't read, continue
 			continue
-		} else if input.Type.T == ArrayTy {
-			// need to move this up because they read sequentially
-			j += input.Type.Size
 		}
 		marshalledValue, err := toGoType((i+j)*32, input.Type, output)
 		if err != nil {
 			return err
 		}
+		if input.Type.T == ArrayTy {
+			// combined index ('i' + 'j') need to be adjusted only by size of array, thus
+			// we need to decrement 'j' because 'i' was incremented
+			j += input.Type.Size - 1
+		}
 		reflectValue := reflect.ValueOf(marshalledValue)
 
 		switch value.Kind() {

accounts/abi/event_test.go

@@ -17,11 +17,14 @@
 package abi
 
 import (
+	"bytes"
+	"reflect"
 	"strings"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/stretchr/testify/require"
 )
 
 func TestEventId(t *testing.T) {
@@ -54,3 +57,23 @@ func TestEventId(t *testing.T) {
 		}
 	}
 }
+
+// TestEventMultiValueWithArrayUnpack verifies that array fields will be counted after parsing array.
+func TestEventMultiValueWithArrayUnpack(t *testing.T) {
+	definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": false, "name":"value1", "type":"uint8[2]"},{"indexed": false, "name":"value2", "type":"uint8"}]}]`
+	type testStruct struct {
+		Value1 [2]uint8
+		Value2 uint8
+	}
+	abi, err := JSON(strings.NewReader(definition))
+	require.NoError(t, err)
+	var b bytes.Buffer
+	var i uint8 = 1
+	for ; i <= 3; i++ {
+		b.Write(packNum(reflect.ValueOf(i)))
+	}
+	var rst testStruct
+	require.NoError(t, abi.Unpack(&rst, "test", b.Bytes()))
+	require.Equal(t, [2]uint8{1, 2}, rst.Value1)
+	require.Equal(t, uint8(3), rst.Value2)
+}

accounts/abi/method.go

@@ -95,14 +95,15 @@ func (method Method) tupleUnpack(v interface{}, output []byte) error {
 	j := 0
 	for i := 0; i < len(method.Outputs); i++ {
 		toUnpack := method.Outputs[i]
-		if toUnpack.Type.T == ArrayTy {
-			// need to move this up because they read sequentially
-			j += toUnpack.Type.Size
-		}
 		marshalledValue, err := toGoType((i+j)*32, toUnpack.Type, output)
 		if err != nil {
 			return err
 		}
+		if toUnpack.Type.T == ArrayTy {
+			// combined index ('i' + 'j') need to be adjusted only by size of array, thus
+			// we need to decrement 'j' because 'i' was incremented
+			j += toUnpack.Type.Size - 1
+		}
 		reflectValue := reflect.ValueOf(marshalledValue)
 
 		switch value.Kind() {

accounts/abi/unpack_test.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"math/big"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -261,25 +262,27 @@ var unpackTests = []unpackTest{
 
 func TestUnpack(t *testing.T) {
 	for i, test := range unpackTests {
-		def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
-		abi, err := JSON(strings.NewReader(def))
+			def := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def)
-		if err != nil {
+			abi, err := JSON(strings.NewReader(def))
-			t.Fatalf("invalid ABI definition %s: %v", def, err)
+			if err != nil {
-		}
+				t.Fatalf("invalid ABI definition %s: %v", def, err)
-		encb, err := hex.DecodeString(test.enc)
+			}
-		if err != nil {
+			encb, err := hex.DecodeString(test.enc)
-			t.Fatalf("invalid hex: %s" + test.enc)
+			if err != nil {
-		}
+				t.Fatalf("invalid hex: %s" + test.enc)
-		outptr := reflect.New(reflect.TypeOf(test.want))
+			}
-		err = abi.Unpack(outptr.Interface(), "method", encb)
+			outptr := reflect.New(reflect.TypeOf(test.want))
-		if err := test.checkError(err); err != nil {
+			err = abi.Unpack(outptr.Interface(), "method", encb)
-			t.Errorf("test %d (%v) failed: %v", i, test.def, err)
+			if err := test.checkError(err); err != nil {
-			continue
+				t.Errorf("test %d (%v) failed: %v", i, test.def, err)
-		}
+				return
-		out := outptr.Elem().Interface()
+			}
-		if !reflect.DeepEqual(test.want, out) {
+			out := outptr.Elem().Interface()
-			t.Errorf("test %d (%v) failed: expected %v, got %v", i, test.def, test.want, out)
+			if !reflect.DeepEqual(test.want, out) {
-		}
+				t.Errorf("test %d (%v) failed: expected %v, got %v", i, test.def, test.want, out)
+			}
+		})
 	}
 }
 
@@ -336,6 +339,29 @@ func TestMultiReturnWithStruct(t *testing.T) {
 	}
 }
 
+func TestMultiReturnWithArray(t *testing.T) {
+	const definition = `[{"name" : "multi", "outputs": [{"type": "uint64[3]"}, {"type": "uint64"}]}]`
+	abi, err := JSON(strings.NewReader(definition))
+	if err != nil {
+		t.Fatal(err)
+	}
+	buff := new(bytes.Buffer)
+	buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000090000000000000000000000000000000000000000000000000000000000000009"))
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000008"))
+
+	ret1, ret1Exp := new([3]uint64), [3]uint64{9, 9, 9}
+	ret2, ret2Exp := new(uint64), uint64(8)
+	if err := abi.Unpack(&[]interface{}{ret1, ret2}, "multi", buff.Bytes()); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(*ret1, ret1Exp) {
+		t.Error("array result", *ret1, "!= Expected", ret1Exp)
+	}
+	if *ret2 != ret2Exp {
+		t.Error("int result", *ret2, "!= Expected", ret2Exp)
+	}
+}
+
 func TestUnmarshal(t *testing.T) {
 	const definition = `[
 	{ "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] },

accounts/keystore/presale.go

@@ -58,6 +58,9 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error
 	if err != nil {
 		return nil, errors.New("invalid hex in encSeed")
 	}
+	if len(encSeedBytes) < 16 {
+		return nil, errors.New("invalid encSeed, too short")
+	}
 	iv := encSeedBytes[:16]
 	cipherText := encSeedBytes[16:]
 	/*

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.9.windows-%GETH_ARCH%.zip
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.9.2.windows-%GETH_ARCH%.zip
-  - 7z x go1.9.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - 7z x go1.9.2.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

cmd/geth/consolecmd.go

@@ -120,8 +120,12 @@ func remoteConsole(ctx *cli.Context) error {
 		if ctx.GlobalIsSet(utils.DataDirFlag.Name) {
 			path = ctx.GlobalString(utils.DataDirFlag.Name)
 		}
-		if path != "" && ctx.GlobalBool(utils.TestnetFlag.Name) {
+		if path != "" {
-			path = filepath.Join(path, "testnet")
+			if ctx.GlobalBool(utils.TestnetFlag.Name) {
+				path = filepath.Join(path, "testnet")
+			} else if ctx.GlobalBool(utils.RinkebyFlag.Name) {
+				path = filepath.Join(path, "rinkeby")
+			}
 		}
 		endpoint = fmt.Sprintf("%s/geth.ipc", path)
 	}

consensus/ethash/consensus.go

@@ -36,9 +36,10 @@ import (
 
 // Ethash proof-of-work protocol constants.
 var (
-	FrontierBlockReward  *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block
+	FrontierBlockReward    *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block
-	ByzantiumBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium
+	ByzantiumBlockReward   *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium
-	maxUncles                     = 2                 // Maximum number of uncles allowed in a single block
+	maxUncles                       = 2                 // Maximum number of uncles allowed in a single block
+	allowedFutureBlockTime          = 15 * time.Second  // Max time from current time allowed for blocks, before they're considered future blocks
 )
 
 // Various error messages to mark blocks invalid. These should be private to
@@ -231,7 +232,7 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent *
 			return errLargeBlockTime
 		}
 	} else {
-		if header.Time.Cmp(big.NewInt(time.Now().Unix())) > 0 {
+		if header.Time.Cmp(big.NewInt(time.Now().Add(allowedFutureBlockTime).Unix())) > 0 {
 			return consensus.ErrFutureBlock
 		}
 	}

console/console.go

@@ -92,6 +92,9 @@ func New(config Config) (*Console, error) {
 		printer:  config.Printer,
 		histPath: filepath.Join(config.DataDir, HistoryFile),
 	}
+	if err := os.MkdirAll(config.DataDir, 0700); err != nil {
+		return nil, err
+	}
 	if err := console.init(config.Preload); err != nil {
 		return nil, err
 	}
@@ -423,7 +426,7 @@ func (c *Console) Execute(path string) error {
 	return c.jsre.Exec(path)
 }
 
-// Stop cleans up the console and terminates the runtime envorinment.
+// Stop cleans up the console and terminates the runtime environment.
 func (c *Console) Stop(graceful bool) error {
 	if err := ioutil.WriteFile(c.histPath, []byte(strings.Join(c.history, "\n")), 0600); err != nil {
 		return err

contracts/release/release.go

@@ -137,7 +137,7 @@ func (r *ReleaseService) checkVersion() {
 	if err != nil {
 		if err == bind.ErrNoCode {
 			log.Debug("Release oracle not found", "contract", r.config.Oracle)
-		} else {
+		} else if err != les.ErrNoPeers {
 			log.Error("Failed to retrieve current release", "err", err)
 		}
 		return

core/tx_pool.go

@@ -103,7 +103,7 @@ var (
 	underpricedTxCounter = metrics.NewCounter("txpool/underpriced")
 )
 
-// TxStatus is the current status of a transaction as seen py the pool.
+// TxStatus is the current status of a transaction as seen by the pool.
 type TxStatus uint
 
 const (
@@ -199,7 +199,7 @@ type TxPool struct {
 	pendingState  *state.ManagedState // Pending state tracking virtual nonces
 	currentMaxGas *big.Int            // Current gas limit for transaction caps
 
-	locals  *accountSet // Set of local transaction to exepmt from evicion rules
+	locals  *accountSet // Set of local transaction to exempt from eviction rules
 	journal *txJournal  // Journal of local transaction to back up to disk
 
 	pending map[common.Address]*txList         // All currently processable transactions
@@ -214,7 +214,7 @@ type TxPool struct {
 }
 
 // NewTxPool creates a new transaction pool to gather, sort and filter inbound
-// trnsactions from the network.
+// transactions from the network.
 func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain blockChain) *TxPool {
 	// Sanitize the input to ensure no vulnerable gas prices are set
 	config = (&config).sanitize()
@@ -360,7 +360,7 @@ func (pool *TxPool) reset(oldHead, newHead *types.Header) {
 		newNum := newHead.Number.Uint64()
 
 		if depth := uint64(math.Abs(float64(oldNum) - float64(newNum))); depth > 64 {
-			log.Warn("Skipping deep transaction reorg", "depth", depth)
+			log.Debug("Skipping deep transaction reorg", "depth", depth)
 		} else {
 			// Reorg seems shallow enough to pull in all transactions into memory
 			var discarded, included types.Transactions
@@ -838,7 +838,7 @@ func (pool *TxPool) Status(hashes []common.Hash) []TxStatus {
 	for i, hash := range hashes {
 		if tx := pool.all[hash]; tx != nil {
 			from, _ := types.Sender(pool.signer, tx) // already validated
-			if pool.pending[from].txs.items[tx.Nonce()] != nil {
+			if pool.pending[from] != nil && pool.pending[from].txs.items[tx.Nonce()] != nil {
 				status[i] = TxStatusPending
 			} else {
 				status[i] = TxStatusQueued

core/tx_pool_test.go

@@ -1563,6 +1563,63 @@ func testTransactionJournaling(t *testing.T, nolocals bool) {
 	pool.Stop()
 }
 
+// TestTransactionStatusCheck tests that the pool can correctly retrieve the
+// pending status of individual transactions.
+func TestTransactionStatusCheck(t *testing.T) {
+	t.Parallel()
+
+	// Create the pool to test the status retrievals with
+	db, _ := ethdb.NewMemDatabase()
+	statedb, _ := state.New(common.Hash{}, state.NewDatabase(db))
+	blockchain := &testBlockChain{statedb, big.NewInt(1000000), new(event.Feed)}
+
+	pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain)
+	defer pool.Stop()
+
+	// Create the test accounts to check various transaction statuses with
+	keys := make([]*ecdsa.PrivateKey, 3)
+	for i := 0; i < len(keys); i++ {
+		keys[i], _ = crypto.GenerateKey()
+		pool.currentState.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000))
+	}
+	// Generate and queue a batch of transactions, both pending and queued
+	txs := types.Transactions{}
+
+	txs = append(txs, pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[0])) // Pending only
+	txs = append(txs, pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[1])) // Pending and queued
+	txs = append(txs, pricedTransaction(2, big.NewInt(100000), big.NewInt(1), keys[1]))
+	txs = append(txs, pricedTransaction(2, big.NewInt(100000), big.NewInt(1), keys[2])) // Queued only
+
+	// Import the transaction and ensure they are correctly added
+	pool.AddRemotes(txs)
+
+	pending, queued := pool.Stats()
+	if pending != 2 {
+		t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2)
+	}
+	if queued != 2 {
+		t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2)
+	}
+	if err := validateTxPoolInternals(pool); err != nil {
+		t.Fatalf("pool internal state corrupted: %v", err)
+	}
+	// Retrieve the status of each transaction and validate them
+	hashes := make([]common.Hash, len(txs))
+	for i, tx := range txs {
+		hashes[i] = tx.Hash()
+	}
+	hashes = append(hashes, common.Hash{})
+
+	statuses := pool.Status(hashes)
+	expect := []TxStatus{TxStatusPending, TxStatusPending, TxStatusQueued, TxStatusQueued, TxStatusUnknown}
+
+	for i := 0; i < len(statuses); i++ {
+		if statuses[i] != expect[i] {
+			t.Errorf("transaction %d: status mismatch: have %v, want %v", i, statuses[i], expect[i])
+		}
+	}
+}
+
 // Benchmarks the speed of validating the contents of the pending queue of the
 // transaction pool.
 func BenchmarkPendingDemotion100(b *testing.B)   { benchmarkPendingDemotion(b, 100) }

core/types/transaction.go

@@ -153,6 +153,7 @@ func (tx *Transaction) DecodeRLP(s *rlp.Stream) error {
 	return err
 }
 
+// MarshalJSON encodes the web3 RPC transaction format.
 func (tx *Transaction) MarshalJSON() ([]byte, error) {
 	hash := tx.Hash()
 	data := tx.data
@@ -168,8 +169,8 @@ func (tx *Transaction) UnmarshalJSON(input []byte) error {
 	}
 	var V byte
 	if isProtectedV(dec.V) {
-		chainId := deriveChainId(dec.V).Uint64()
+		chainID := deriveChainId(dec.V).Uint64()
-		V = byte(dec.V.Uint64() - 35 - 2*chainId)
+		V = byte(dec.V.Uint64() - 35 - 2*chainID)
 	} else {
 		V = byte(dec.V.Uint64() - 27)
 	}
@@ -192,10 +193,9 @@ func (tx *Transaction) CheckNonce() bool   { return true }
 func (tx *Transaction) To() *common.Address {
 	if tx.data.Recipient == nil {
 		return nil
-	} else {
-		to := *tx.data.Recipient
-		return &to
 	}
+	to := *tx.data.Recipient
+	return &to
 }
 
 // Hash hashes the RLP encoding of tx.
@@ -315,22 +315,22 @@ func (tx *Transaction) String() string {
 	)
 }
 
-// Transaction slice type for basic sorting.
+// Transactions is a Transaction slice type for basic sorting.
 type Transactions []*Transaction
 
-// Len returns the length of s
+// Len returns the length of s.
 func (s Transactions) Len() int { return len(s) }
 
-// Swap swaps the i'th and the j'th element in s
+// Swap swaps the i'th and the j'th element in s.
 func (s Transactions) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
 
-// GetRlp implements Rlpable and returns the i'th element of s in rlp
+// GetRlp implements Rlpable and returns the i'th element of s in rlp.
 func (s Transactions) GetRlp(i int) []byte {
 	enc, _ := rlp.EncodeToBytes(s[i])
 	return enc
 }
 
-// Returns a new set t which is the difference between a to b
+// TxDifference returns a new set t which is the difference between a to b.
 func TxDifference(a, b Transactions) (keep Transactions) {
 	keep = make(Transactions, 0, len(a))
 
@@ -378,7 +378,7 @@ func (s *TxByPrice) Pop() interface{} {
 }
 
 // TransactionsByPriceAndNonce represents a set of transactions that can return
-// transactions in a profit-maximising sorted order, while supporting removing
+// transactions in a profit-maximizing sorted order, while supporting removing
 // entire batches of transactions for non-executable accounts.
 type TransactionsByPriceAndNonce struct {
 	txs    map[common.Address]Transactions // Per account nonce-sorted list of transactions

core/vm/evm.go

@@ -38,7 +38,7 @@ type (
 )
 
 // run runs the given contract and takes care of running precompiles with a fallback to the byte code interpreter.
-func run(evm *EVM, snapshot int, contract *Contract, input []byte) ([]byte, error) {
+func run(evm *EVM, contract *Contract, input []byte) ([]byte, error) {
 	if contract.CodeAddr != nil {
 		precompiles := PrecompiledContractsHomestead
 		if evm.ChainConfig().IsByzantium(evm.BlockNumber) {
@@ -48,7 +48,7 @@ func run(evm *EVM, snapshot int, contract *Contract, input []byte) ([]byte, erro
 			return RunPrecompiledContract(p, input, contract)
 		}
 	}
-	return evm.interpreter.Run(snapshot, contract, input)
+	return evm.interpreter.Run(contract, input)
 }
 
 // Context provides the EVM with auxiliary information. Once provided
@@ -171,7 +171,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
 	contract := NewContract(caller, to, value, gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
-	ret, err = run(evm, snapshot, contract, input)
+	ret, err = run(evm, contract, input)
 	// When an error was returned by the EVM or when setting the creation code
 	// above we revert to the snapshot and consume any gas remaining. Additionally
 	// when we're in homestead this also counts for code storage gas errors.
@@ -215,7 +215,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte,
 	contract := NewContract(caller, to, value, gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
-	ret, err = run(evm, snapshot, contract, input)
+	ret, err = run(evm, contract, input)
 	if err != nil {
 		evm.StateDB.RevertToSnapshot(snapshot)
 		if err != errExecutionReverted {
@@ -248,7 +248,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by
 	contract := NewContract(caller, to, nil, gas).AsDelegate()
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
-	ret, err = run(evm, snapshot, contract, input)
+	ret, err = run(evm, contract, input)
 	if err != nil {
 		evm.StateDB.RevertToSnapshot(snapshot)
 		if err != errExecutionReverted {
@@ -291,7 +291,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
 	// When an error was returned by the EVM or when setting the creation code
 	// above we revert to the snapshot and consume any gas remaining. Additionally
 	// when we're in Homestead this also counts for code storage gas errors.
-	ret, err = run(evm, snapshot, contract, input)
+	ret, err = run(evm, contract, input)
 	if err != nil {
 		evm.StateDB.RevertToSnapshot(snapshot)
 		if err != errExecutionReverted {
@@ -338,7 +338,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I
 	if evm.vmConfig.NoRecursion && evm.depth > 0 {
 		return nil, contractAddr, gas, nil
 	}
-	ret, err = run(evm, snapshot, contract, nil)
+	ret, err = run(evm, contract, nil)
 	// check whether the max code size has been exceeded
 	maxCodeSizeExceeded := evm.ChainConfig().IsEIP158(evm.BlockNumber) && len(ret) > params.MaxCodeSize
 	// if the contract creation ran successfully and no errors were returned

core/vm/interpreter.go

@@ -107,9 +107,9 @@ func (in *Interpreter) enforceRestrictions(op OpCode, operation operation, stack
 // the return byte-slice and an error if one occurred.
 //
 // It's important to note that any errors returned by the interpreter should be
-// considered a revert-and-consume-all-gas operation. No error specific checks
+// considered a revert-and-consume-all-gas operation except for
-// should be handled to reduce complexity and errors further down the in.
+// errExecutionReverted which means revert-and-keep-gas-left.
-func (in *Interpreter) Run(snapshot int, contract *Contract, input []byte) (ret []byte, err error) {
+func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err error) {
 	// Increment the call depth which is restricted to 1024
 	in.evm.depth++
 	defer func() { in.evm.depth-- }()

crypto/crypto.go

@@ -79,7 +79,7 @@ func ToECDSA(d []byte) (*ecdsa.PrivateKey, error) {
 	return toECDSA(d, true)
 }
 
-// ToECDSAUnsafe blidly converts a binary blob to a private key. It should almost
+// ToECDSAUnsafe blindly converts a binary blob to a private key. It should almost
 // never be used unless you are sure the input is valid and want to avoid hitting
 // errors due to bad origin encoding (0 prefixes cut off).
 func ToECDSAUnsafe(d []byte) *ecdsa.PrivateKey {

crypto/secp256k1/curve.go

@@ -34,7 +34,6 @@ package secp256k1
 import (
 	"crypto/elliptic"
 	"math/big"
-	"sync"
 	"unsafe"
 
 	"github.com/ethereum/go-ethereum/common/math"
@@ -42,7 +41,7 @@ import (
 
 /*
 #include "libsecp256k1/include/secp256k1.h"
-extern int secp256k1_pubkey_scalar_mul(const secp256k1_context* ctx, const unsigned char *point, const unsigned char *scalar);
+extern int secp256k1_ext_scalar_mul(const secp256k1_context* ctx, const unsigned char *point, const unsigned char *scalar);
 */
 import "C"
 
@@ -236,7 +235,7 @@ func (BitCurve *BitCurve) ScalarMult(Bx, By *big.Int, scalar []byte) (*big.Int,
 	math.ReadBits(By, point[32:])
 	pointPtr := (*C.uchar)(unsafe.Pointer(&point[0]))
 	scalarPtr := (*C.uchar)(unsafe.Pointer(&scalar[0]))
-	res := C.secp256k1_pubkey_scalar_mul(context, pointPtr, scalarPtr)
+	res := C.secp256k1_ext_scalar_mul(context, pointPtr, scalarPtr)
 
 	// Unpack the result and clear temporaries.
 	x := new(big.Int).SetBytes(point[:32])
@@ -263,14 +262,10 @@ func (BitCurve *BitCurve) ScalarBaseMult(k []byte) (*big.Int, *big.Int) {
 // X9.62.
 func (BitCurve *BitCurve) Marshal(x, y *big.Int) []byte {
 	byteLen := (BitCurve.BitSize + 7) >> 3
-
 	ret := make([]byte, 1+2*byteLen)
-	ret[0] = 4 // uncompressed point
+	ret[0] = 4 // uncompressed point flag
-
+	math.ReadBits(x, ret[1:1+byteLen])
-	xBytes := x.Bytes()
+	math.ReadBits(y, ret[1+byteLen:])
-	copy(ret[1+byteLen-len(xBytes):], xBytes)
-	yBytes := y.Bytes()
-	copy(ret[1+2*byteLen-len(yBytes):], yBytes)
 	return ret
 }
 
@@ -289,24 +284,21 @@ func (BitCurve *BitCurve) Unmarshal(data []byte) (x, y *big.Int) {
 	return
 }
 
-var (
+var theCurve = new(BitCurve)
-	initonce sync.Once
-	theCurve *BitCurve
-)
 
-// S256 returns a BitCurve which implements secp256k1 (see SEC 2 section 2.7.1)
+func init() {
+	// See SEC 2 section 2.7.1
+	// curve parameters taken from:
+	// http://www.secg.org/collateral/sec2_final.pdf
+	theCurve.P, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", 16)
+	theCurve.N, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", 16)
+	theCurve.B, _ = new(big.Int).SetString("0000000000000000000000000000000000000000000000000000000000000007", 16)
+	theCurve.Gx, _ = new(big.Int).SetString("79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798", 16)
+	theCurve.Gy, _ = new(big.Int).SetString("483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8", 16)
+	theCurve.BitSize = 256
+}
+
+// S256 returns a BitCurve which implements secp256k1.
 func S256() *BitCurve {
-	initonce.Do(func() {
-		// See SEC 2 section 2.7.1
-		// curve parameters taken from:
-		// http://www.secg.org/collateral/sec2_final.pdf
-		theCurve = new(BitCurve)
-		theCurve.P, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", 16)
-		theCurve.N, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", 16)
-		theCurve.B, _ = new(big.Int).SetString("0000000000000000000000000000000000000000000000000000000000000007", 16)
-		theCurve.Gx, _ = new(big.Int).SetString("79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798", 16)
-		theCurve.Gy, _ = new(big.Int).SetString("483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8", 16)
-		theCurve.BitSize = 256
-	})
 	return theCurve
 }

crypto/secp256k1/ext.h

@@ -19,7 +19,7 @@ static secp256k1_context* secp256k1_context_create_sign_verify() {
 	return secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY);
 }
 
-// secp256k1_ecdsa_recover_pubkey recovers the public key of an encoded compact signature.
+// secp256k1_ext_ecdsa_recover recovers the public key of an encoded compact signature.
 //
 // Returns: 1: recovery was successful
 //          0: recovery was not successful
@@ -27,7 +27,7 @@ static secp256k1_context* secp256k1_context_create_sign_verify() {
 //  Out:    pubkey_out: the serialized 65-byte public key of the signer (cannot be NULL)
 //  In:     sigdata:    pointer to a 65-byte signature with the recovery id at the end (cannot be NULL)
 //          msgdata:    pointer to a 32-byte message (cannot be NULL)
-static int secp256k1_ecdsa_recover_pubkey(
+static int secp256k1_ext_ecdsa_recover(
 	const secp256k1_context* ctx,
 	unsigned char *pubkey_out,
 	const unsigned char *sigdata,
@@ -46,7 +46,7 @@ static int secp256k1_ecdsa_recover_pubkey(
 	return secp256k1_ec_pubkey_serialize(ctx, pubkey_out, &outputlen, &pubkey, SECP256K1_EC_UNCOMPRESSED);
 }
 
-// secp256k1_ecdsa_verify_enc verifies an encoded compact signature.
+// secp256k1_ext_ecdsa_verify verifies an encoded compact signature.
 //
 // Returns: 1: signature is valid
 //          0: signature is invalid
@@ -55,7 +55,7 @@ static int secp256k1_ecdsa_recover_pubkey(
 //          msgdata:    pointer to a 32-byte message (cannot be NULL)
 //          pubkeydata: pointer to public key data (cannot be NULL)
 //          pubkeylen:  length of pubkeydata
-static int secp256k1_ecdsa_verify_enc(
+static int secp256k1_ext_ecdsa_verify(
 	const secp256k1_context* ctx,
 	const unsigned char *sigdata,
 	const unsigned char *msgdata,
@@ -74,28 +74,34 @@ static int secp256k1_ecdsa_verify_enc(
 	return secp256k1_ecdsa_verify(ctx, &sig, msgdata, &pubkey);
 }
 
-// secp256k1_decompress_pubkey decompresses a public key.
+// secp256k1_ext_reencode_pubkey decodes then encodes a public key. It can be used to
+// convert between public key formats. The input/output formats are chosen depending on the
+// length of the input/output buffers.
 //
-// Returns: 1: public key is valid
+// Returns: 1: conversion successful
-//          0: public key is invalid
+//          0: conversion unsuccessful
 // Args:    ctx:        pointer to a context object (cannot be NULL)
-//  Out:    pubkey_out: the serialized 65-byte public key (cannot be NULL)
+//  Out:    out:        output buffer that will contain the reencoded key (cannot be NULL)
-//  In:     pubkeydata: pointer to 33 bytes of compressed public key data (cannot be NULL)
+//  In:     outlen:     length of out (33 for compressed keys, 65 for uncompressed keys)
-static int secp256k1_decompress_pubkey(
+//          pubkeydata: the input public key (cannot be NULL)
+//          pubkeylen:  length of pubkeydata
+static int secp256k1_ext_reencode_pubkey(
 	const secp256k1_context* ctx,
-	unsigned char *pubkey_out,
+	unsigned char *out,
-	const unsigned char *pubkeydata
+	size_t outlen,
+	const unsigned char *pubkeydata,
+	size_t pubkeylen
 ) {
 	secp256k1_pubkey pubkey;
 
-	if (!secp256k1_ec_pubkey_parse(ctx, &pubkey, pubkeydata, 33)) {
+	if (!secp256k1_ec_pubkey_parse(ctx, &pubkey, pubkeydata, pubkeylen)) {
 		return 0;
 	}
-	size_t outputlen = 65;
+	unsigned int flag = (outlen == 33) ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED;
-	return secp256k1_ec_pubkey_serialize(ctx, pubkey_out, &outputlen, &pubkey, SECP256K1_EC_UNCOMPRESSED);
+	return secp256k1_ec_pubkey_serialize(ctx, out, &outlen, &pubkey, flag);
 }
 
-// secp256k1_pubkey_scalar_mul multiplies a point by a scalar in constant time.
+// secp256k1_ext_scalar_mul multiplies a point by a scalar in constant time.
 //
 // Returns: 1: multiplication was successful
 //          0: scalar was invalid (zero or overflow)
@@ -104,7 +110,7 @@ static int secp256k1_decompress_pubkey(
 //  In:     point:    pointer to a 64-byte public point,
 //                    encoded as two 256bit big-endian numbers.
 //          scalar:   a 32-byte scalar with which to multiply the point
-int secp256k1_pubkey_scalar_mul(const secp256k1_context* ctx, unsigned char *point, const unsigned char *scalar) {
+int secp256k1_ext_scalar_mul(const secp256k1_context* ctx, unsigned char *point, const unsigned char *scalar) {
 	int ret = 0;
 	int overflow = 0;
 	secp256k1_fe feX, feY;

crypto/secp256k1/secp256.go

@@ -115,7 +115,7 @@ func RecoverPubkey(msg []byte, sig []byte) ([]byte, error) {
 		sigdata = (*C.uchar)(unsafe.Pointer(&sig[0]))
 		msgdata = (*C.uchar)(unsafe.Pointer(&msg[0]))
 	)
-	if C.secp256k1_ecdsa_recover_pubkey(context, (*C.uchar)(unsafe.Pointer(&pubkey[0])), sigdata, msgdata) == 0 {
+	if C.secp256k1_ext_ecdsa_recover(context, (*C.uchar)(unsafe.Pointer(&pubkey[0])), sigdata, msgdata) == 0 {
 		return nil, ErrRecoverFailed
 	}
 	return pubkey, nil
@@ -130,22 +130,42 @@ func VerifySignature(pubkey, msg, signature []byte) bool {
 	sigdata := (*C.uchar)(unsafe.Pointer(&signature[0]))
 	msgdata := (*C.uchar)(unsafe.Pointer(&msg[0]))
 	keydata := (*C.uchar)(unsafe.Pointer(&pubkey[0]))
-	return C.secp256k1_ecdsa_verify_enc(context, sigdata, msgdata, keydata, C.size_t(len(pubkey))) != 0
+	return C.secp256k1_ext_ecdsa_verify(context, sigdata, msgdata, keydata, C.size_t(len(pubkey))) != 0
 }
 
 // DecompressPubkey parses a public key in the 33-byte compressed format.
 // It returns non-nil coordinates if the public key is valid.
-func DecompressPubkey(pubkey []byte) (X, Y *big.Int) {
+func DecompressPubkey(pubkey []byte) (x, y *big.Int) {
 	if len(pubkey) != 33 {
 		return nil, nil
 	}
-	buf := make([]byte, 65)
+	var (
-	bufdata := (*C.uchar)(unsafe.Pointer(&buf[0]))
+		pubkeydata = (*C.uchar)(unsafe.Pointer(&pubkey[0]))
-	pubkeydata := (*C.uchar)(unsafe.Pointer(&pubkey[0]))
+		pubkeylen  = C.size_t(len(pubkey))
-	if C.secp256k1_decompress_pubkey(context, bufdata, pubkeydata) == 0 {
+		out        = make([]byte, 65)
+		outdata    = (*C.uchar)(unsafe.Pointer(&out[0]))
+		outlen     = C.size_t(len(out))
+	)
+	if C.secp256k1_ext_reencode_pubkey(context, outdata, outlen, pubkeydata, pubkeylen) == 0 {
 		return nil, nil
 	}
-	return new(big.Int).SetBytes(buf[1:33]), new(big.Int).SetBytes(buf[33:])
+	return new(big.Int).SetBytes(out[1:33]), new(big.Int).SetBytes(out[33:])
+}
+
+// CompressPubkey encodes a public key to 33-byte compressed format.
+func CompressPubkey(x, y *big.Int) []byte {
+	var (
+		pubkey     = S256().Marshal(x, y)
+		pubkeydata = (*C.uchar)(unsafe.Pointer(&pubkey[0]))
+		pubkeylen  = C.size_t(len(pubkey))
+		out        = make([]byte, 33)
+		outdata    = (*C.uchar)(unsafe.Pointer(&out[0]))
+		outlen     = C.size_t(len(out))
+	)
+	if C.secp256k1_ext_reencode_pubkey(context, outdata, outlen, pubkeydata, pubkeylen) == 0 {
+		panic("libsecp256k1 error")
+	}
+	return out
 }
 
 func checkSignature(sig []byte) error {

crypto/signature_cgo.go

@@ -76,6 +76,11 @@ func DecompressPubkey(pubkey []byte) (*ecdsa.PublicKey, error) {
 	return &ecdsa.PublicKey{X: x, Y: y, Curve: S256()}, nil
 }
 
+// CompressPubkey encodes a public key to the 33-byte compressed format.
+func CompressPubkey(pubkey *ecdsa.PublicKey) []byte {
+	return secp256k1.CompressPubkey(pubkey.X, pubkey.Y)
+}
+
 // S256 returns an instance of the secp256k1 curve.
 func S256() elliptic.Curve {
 	return secp256k1.S256()

crypto/signature_nocgo.go

@@ -87,6 +87,10 @@ func VerifySignature(pubkey, hash, signature []byte) bool {
 	if err != nil {
 		return false
 	}
+	// Reject malleable signatures. libsecp256k1 does this check but btcec doesn't.
+	if sig.S.Cmp(secp256k1_halfN) > 0 {
+		return false
+	}
 	return sig.Verify(hash, key)
 }
 
@@ -102,6 +106,11 @@ func DecompressPubkey(pubkey []byte) (*ecdsa.PublicKey, error) {
 	return key.ToECDSA(), nil
 }
 
+// CompressPubkey encodes a public key to the 33-byte compressed format.
+func CompressPubkey(pubkey *ecdsa.PublicKey) []byte {
+	return (*btcec.PublicKey)(pubkey).SerializeCompressed()
+}
+
 // S256 returns an instance of the secp256k1 curve.
 func S256() elliptic.Curve {
 	return btcec.S256()

crypto/signature_test.go

@@ -18,10 +18,13 @@ package crypto
 
 import (
 	"bytes"
+	"crypto/ecdsa"
+	"reflect"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/common/math"
 )
 
 var (
@@ -65,6 +68,21 @@ func TestVerifySignature(t *testing.T) {
 	if VerifySignature(testpubkey, testmsg, sig[:len(sig)-2]) {
 		t.Errorf("signature valid even though it's incomplete")
 	}
+	wrongkey := common.CopyBytes(testpubkey)
+	wrongkey[10]++
+	if VerifySignature(wrongkey, testmsg, sig) {
+		t.Errorf("signature valid with with wrong public key")
+	}
+}
+
+// This test checks that VerifySignature rejects malleable signatures with s > N/2.
+func TestVerifySignatureMalleable(t *testing.T) {
+	sig := hexutil.MustDecode("0x638a54215d80a6713c8d523a6adc4e6e73652d859103a36b700851cb0e61b66b8ebfc1a610c57d732ec6e0a8f06a9a7a28df5051ece514702ff9cdff0b11f454")
+	key := hexutil.MustDecode("0x03ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd3138")
+	msg := hexutil.MustDecode("0xd301ce462d3e639518f482c7f03821fec1e602018630ce621e1e7851c12343a6")
+	if VerifySignature(key, msg, sig) {
+		t.Error("VerifySignature returned true for malleable signature")
+	}
 }
 
 func TestDecompressPubkey(t *testing.T) {
@@ -86,6 +104,36 @@ func TestDecompressPubkey(t *testing.T) {
 	}
 }
 
+func TestCompressPubkey(t *testing.T) {
+	key := &ecdsa.PublicKey{
+		Curve: S256(),
+		X:     math.MustParseBig256("0xe32df42865e97135acfb65f3bae71bdc86f4d49150ad6a440b6f15878109880a"),
+		Y:     math.MustParseBig256("0x0a2b2667f7e725ceea70c673093bf67663e0312623c8e091b13cf2c0f11ef652"),
+	}
+	compressed := CompressPubkey(key)
+	if !bytes.Equal(compressed, testpubkeyc) {
+		t.Errorf("wrong public key result: got %x, want %x", compressed, testpubkeyc)
+	}
+}
+
+func TestPubkeyRandom(t *testing.T) {
+	const runs = 200
+
+	for i := 0; i < runs; i++ {
+		key, err := GenerateKey()
+		if err != nil {
+			t.Fatalf("iteration %d: %v", i, err)
+		}
+		pubkey2, err := DecompressPubkey(CompressPubkey(&key.PublicKey))
+		if err != nil {
+			t.Fatalf("iteration %d: %v", i, err)
+		}
+		if !reflect.DeepEqual(key.PublicKey, *pubkey2) {
+			t.Fatalf("iteration %d: keys not equal", i)
+		}
+	}
+}
+
 func BenchmarkEcrecoverSignature(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		if _, err := Ecrecover(testmsg, testsig); err != nil {

internal/ethapi/api.go

@@ -17,6 +17,7 @@
 package ethapi
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -1003,9 +1004,12 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context,
 func (s *PublicTransactionPoolAPI) GetTransactionReceipt(hash common.Hash) (map[string]interface{}, error) {
 	tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash)
 	if tx == nil {
-		return nil, nil
+		return nil, errors.New("unknown transaction")
 	}
 	receipt, _, _, _ := core.GetReceipt(s.b.ChainDb(), hash) // Old receipts don't have the lookup data available
+	if receipt == nil {
+		return nil, errors.New("unknown receipt")
+	}
 
 	var signer types.Signer = types.FrontierSigner{}
 	if tx.Protected() {
@@ -1067,8 +1071,11 @@ type SendTxArgs struct {
 	Gas      *hexutil.Big    `json:"gas"`
 	GasPrice *hexutil.Big    `json:"gasPrice"`
 	Value    *hexutil.Big    `json:"value"`
-	Data     hexutil.Bytes   `json:"data"`
 	Nonce    *hexutil.Uint64 `json:"nonce"`
+	// We accept "data" and "input" for backwards-compatibility reasons. "input" is the
+	// newer name and should be preferred by clients.
+	Data  *hexutil.Bytes `json:"data"`
+	Input *hexutil.Bytes `json:"input"`
 }
 
 // setDefaults is a helper function that fills in default values for unspecified tx fields.
@@ -1093,14 +1100,23 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error {
 		}
 		args.Nonce = (*hexutil.Uint64)(&nonce)
 	}
+	if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) {
+		return errors.New(`Both "data" and "input" are set and not equal. Please use "input" to pass transaction call data.`)
+	}
 	return nil
 }
 
 func (args *SendTxArgs) toTransaction() *types.Transaction {
-	if args.To == nil {
+	var input []byte
-		return types.NewContractCreation(uint64(*args.Nonce), (*big.Int)(args.Value), (*big.Int)(args.Gas), (*big.Int)(args.GasPrice), args.Data)
+	if args.Data != nil {
+		input = *args.Data
+	} else if args.Input != nil {
+		input = *args.Input
 	}
-	return types.NewTransaction(uint64(*args.Nonce), *args.To, (*big.Int)(args.Value), (*big.Int)(args.Gas), (*big.Int)(args.GasPrice), args.Data)
+	if args.To == nil {
+		return types.NewContractCreation(uint64(*args.Nonce), (*big.Int)(args.Value), (*big.Int)(args.Gas), (*big.Int)(args.GasPrice), input)
+	}
+	return types.NewTransaction(uint64(*args.Nonce), *args.To, (*big.Int)(args.Value), (*big.Int)(args.Gas), (*big.Int)(args.GasPrice), input)
 }
 
 // submitTransaction is a helper function that submits tx to txPool and logs a message.

internal/ethapi/tracer_test.go

@@ -48,7 +48,7 @@ func runTrace(tracer *JavascriptTracer) (interface{}, error) {
 	contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000)
 	contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0}
 
-	_, err := env.Interpreter().Run(0, contract, []byte{})
+	_, err := env.Interpreter().Run(contract, []byte{})
 	if err != nil {
 		return nil, err
 	}

light/trie.go

@@ -151,7 +151,7 @@ func (t *odrTrie) do(key []byte, fn func() error) error {
 		}
 		r := &TrieRequest{Id: t.id, Key: key}
 		if err := t.db.backend.Retrieve(t.db.ctx, r); err != nil {
-			return fmt.Errorf("can't fetch trie key %x: %v", key, err)
+			return err
 		}
 	}
 }

p2p/discover/database.go

@@ -226,14 +226,14 @@ func (db *nodeDB) ensureExpirer() {
 // expirer should be started in a go routine, and is responsible for looping ad
 // infinitum and dropping stale data from the database.
 func (db *nodeDB) expirer() {
-	tick := time.Tick(nodeDBCleanupCycle)
+	tick := time.NewTicker(nodeDBCleanupCycle)
+	defer tick.Stop()
 	for {
 		select {
-		case <-tick:
+		case <-tick.C:
 			if err := db.expireNodes(); err != nil {
 				log.Error("Failed to expire nodedb items", "err", err)
 			}
-
 		case <-db.quit:
 			return
 		}

swarm/api/api.go

@@ -83,7 +83,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
 
 	// if the URI is immutable, check if the address is a hash
 	isHash := hashMatcher.MatchString(uri.Addr)
-	if uri.Immutable() {
+	if uri.Immutable() || uri.DeprecatedImmutable() {
 		if !isHash {
 			return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr)
 		}

swarm/api/api_test.go

@@ -216,7 +216,7 @@ func TestAPIResolve(t *testing.T) {
 			api := &Api{dns: x.dns}
 			uri := &URI{Addr: x.addr, Scheme: "bzz"}
 			if x.immutable {
-				uri.Scheme = "bzzi"
+				uri.Scheme = "bzz-immutable"
 			}
 			res, err := api.Resolve(uri)
 			if err == nil {

swarm/api/client/client.go

@@ -57,7 +57,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
 	if size <= 0 {
 		return "", errors.New("data size must be greater than zero")
 	}
-	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/", r)
 	if err != nil {
 		return "", err
 	}
@@ -79,7 +79,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
 
 // DownloadRaw downloads raw data from swarm
 func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
-	uri := c.Gateway + "/bzzr:/" + hash
+	uri := c.Gateway + "/bzz-raw:/" + hash
 	res, err := http.DefaultClient.Get(uri)
 	if err != nil {
 		return nil, err
@@ -269,7 +269,7 @@ func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
 //
 // where entries ending with "/" are common prefixes.
 func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
-	res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
+	res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix)
 	if err != nil {
 		return nil, err
 	}

swarm/api/http/roundtripper.go

@@ -35,8 +35,8 @@ import (
 client := httpclient.New()
 // for (private) swarm proxy running locally
 client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
-client.RegisterScheme("bzzi", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzz-immutable", &http.RoundTripper{Port: port})
-client.RegisterScheme("bzzr", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzz-raw", &http.RoundTripper{Port: port})
 
 The port you give the Roundtripper is the port the swarm proxy is listening on.
 If Host is left empty, localhost is assumed.

swarm/api/http/server.go

@@ -86,7 +86,7 @@ type Request struct {
 	uri *api.URI
 }
 
-// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request
+// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
 // body in swarm and returns the resulting storage key as a text/plain response
 func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
 	if r.uri.Path != "" {
@@ -290,7 +290,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
 	fmt.Fprint(w, newKey)
 }
 
-// HandleGetRaw handles a GET request to bzzr://<key> and responds with
+// HandleGetRaw handles a GET request to bzz-raw://<key> and responds with
 // the raw content stored at the given storage key
 func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) {
 	key, err := s.api.Resolve(r.uri)
@@ -424,14 +424,13 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
 	}
 }
 
-// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has
+// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
-// the "list" query parameter set to "true" and returns a list of all files
+// a list of all files contained in <manifest> under <path> grouped into
-// contained in <manifest> under <path> grouped into common prefixes using
+// common prefixes using "/" as a delimiter
-// "/" as a delimiter
 func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	// ensure the root path has a trailing slash so that relative URLs work
 	if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
-		http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently)
+		http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
 		return
 	}
 
@@ -453,7 +452,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	if strings.Contains(r.Header.Get("Accept"), "text/html") {
 		w.Header().Set("Content-Type", "text/html")
 		err := htmlListTemplate.Execute(w, &htmlListData{
-			URI:  r.uri,
+			URI: &api.URI{
+				Scheme: "bzz",
+				Addr:   r.uri.Addr,
+				Path:   r.uri.Path,
+			},
 			List: &list,
 		})
 		if err != nil {
@@ -589,7 +592,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case "POST":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			s.HandlePostRaw(w, req)
 		} else {
 			s.HandlePostFiles(w, req)
@@ -601,7 +604,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		//   new manifest leaving the existing one intact, so it isn't
 		//   strictly a traditional PUT request which replaces content
 		//   at a URI, and POST is more ubiquitous)
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
 			return
 		} else {
@@ -609,28 +612,28 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 
 	case "DELETE":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
 			return
 		}
 		s.HandleDelete(w, req)
 
 	case "GET":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			s.HandleGetRaw(w, req)
 			return
 		}
 
+		if uri.List() {
+			s.HandleGetList(w, req)
+			return
+		}
+
 		if r.Header.Get("Accept") == "application/x-tar" {
 			s.HandleGetFiles(w, req)
 			return
 		}
 
-		if r.URL.Query().Get("list") == "true" {
-			s.HandleGetList(w, req)
-			return
-		}
-
 		s.HandleGetFile(w, req)
 
 	default:

swarm/api/http/server_test.go

@@ -70,7 +70,7 @@ func TestBzzrGetPath(t *testing.T) {
 		wg.Wait()
 	}
 
-	_, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
+	_, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a")
 	if err != nil {
 		t.Fatalf("Failed to connect to proxy: %v", err)
 	}
@@ -79,7 +79,7 @@ func TestBzzrGetPath(t *testing.T) {
 		var resp *http.Response
 		var respbody []byte
 
-		url := srv.URL + "/bzzr:/"
+		url := srv.URL + "/bzz-raw:/"
 		if k[:] != "" {
 			url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
 		}
@@ -104,16 +104,106 @@ func TestBzzrGetPath(t *testing.T) {
 		}
 	}
 
+	for _, c := range []struct {
+		path string
+		json string
+		html string
+	}{
+		{
+			path: "/",
+			json: `{"common_prefixes":["a/"]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\t<tr>\n\t  <td><a href=\"a/\">a/</a></td>\n\t  <td>DIR</td>\n\t  <td>-</td>\n\t</tr>\n      \n\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/a/",
+			json: `{"common_prefixes":["a/b/"],"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/a","mod_time":"0001-01-01T00:00:00Z"}]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\t<tr>\n\t  <td><a href=\"b/\">b/</a></td>\n\t  <td>DIR</td>\n\t  <td>-</td>\n\t</tr>\n      \n\n      \n\t<tr>\n\t  <td><a href=\"a\">a</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/a/b/",
+			json: `{"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/b","mod_time":"0001-01-01T00:00:00Z"},{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/c","mod_time":"0001-01-01T00:00:00Z"}]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\n      \n\t<tr>\n\t  <td><a href=\"b\">b</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n\t<tr>\n\t  <td><a href=\"c\">c</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/x",
+		},
+		{
+			path: "",
+		},
+	} {
+		k := c.path
+		url := srv.URL + "/bzz-list:/"
+		if k[:] != "" {
+			url += common.ToHex(key[0])[2:] + "/" + k[1:]
+		}
+		t.Run("json list "+c.path, func(t *testing.T) {
+			resp, err := http.Get(url)
+			if err != nil {
+				t.Fatalf("HTTP request: %v", err)
+			}
+			defer resp.Body.Close()
+			respbody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("Read response body: %v", err)
+			}
+
+			body := strings.TrimSpace(string(respbody))
+			if body != c.json {
+				isexpectedfailrequest := false
+
+				for _, r := range expectedfailrequests {
+					if k[:] == r {
+						isexpectedfailrequest = true
+					}
+				}
+				if !isexpectedfailrequest {
+					t.Errorf("Response list body %q does not match, expected: %v, got %v", k, c.json, body)
+				}
+			}
+		})
+		t.Run("html list "+c.path, func(t *testing.T) {
+			req, err := http.NewRequest(http.MethodGet, url, nil)
+			if err != nil {
+				t.Fatalf("New request: %v", err)
+			}
+			req.Header.Set("Accept", "text/html")
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				t.Fatalf("HTTP request: %v", err)
+			}
+			defer resp.Body.Close()
+			respbody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("Read response body: %v", err)
+			}
+
+			if string(respbody) != c.html {
+				isexpectedfailrequest := false
+
+				for _, r := range expectedfailrequests {
+					if k[:] == r {
+						isexpectedfailrequest = true
+					}
+				}
+				if !isexpectedfailrequest {
+					t.Errorf("Response list body %q does not match, expected: %q, got %q", k, c.html, string(respbody))
+				}
+			}
+		})
+	}
+
 	nonhashtests := []string{
 		srv.URL + "/bzz:/name",
-		srv.URL + "/bzzi:/nonhash",
+		srv.URL + "/bzz-immutable:/nonhash",
-		srv.URL + "/bzzr:/nonhash",
+		srv.URL + "/bzz-raw:/nonhash",
+		srv.URL + "/bzz-list:/nonhash",
 	}
 
 	nonhashresponses := []string{
 		"error resolving name: no DNS to resolve name: &#34;name&#34;",
 		"error resolving nonhash: immutable address not a content hash: &#34;nonhash&#34;",
 		"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
+		"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
 	}
 
 	for i, url := range nonhashtests {

swarm/api/http/templates.go

@@ -52,7 +52,7 @@ var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.Fu
     <tbody>
       {{ range .List.CommonPrefixes }}
 	<tr>
-	  <td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td>
+	  <td><a href="{{ basename . }}/">{{ basename . }}/</a></td>
 	  <td>DIR</td>
 	  <td>-</td>
 	</tr>

swarm/api/uri.go

@@ -26,7 +26,13 @@ import (
 type URI struct {
 	// Scheme has one of the following values:
 	//
-	// * bzz  - an entry in a swarm manifest
+	// * bzz           - an entry in a swarm manifest
+	// * bzz-raw       - raw swarm content
+	// * bzz-immutable - immutable URI of an entry in a swarm manifest
+	//                   (address is not resolved)
+	// * bzz-list      -  list of all files contained in a swarm manifest
+	//
+	// Deprecated Schemes:
 	// * bzzr - raw swarm content
 	// * bzzi - immutable URI of an entry in a swarm manifest
 	//          (address is not resolved)
@@ -50,7 +56,8 @@ type URI struct {
 // * <scheme>://<addr>
 // * <scheme>://<addr>/<path>
 //
-// with scheme one of bzz, bzzr or bzzi
+// with scheme one of bzz, bzz-raw, bzz-immutable or bzz-list
+// or deprecated ones bzzr and bzzi
 func Parse(rawuri string) (*URI, error) {
 	u, err := url.Parse(rawuri)
 	if err != nil {
@@ -60,7 +67,7 @@ func Parse(rawuri string) (*URI, error) {
 
 	// check the scheme is valid
 	switch uri.Scheme {
-	case "bzz", "bzzi", "bzzr":
+	case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzzr", "bzzi":
 	default:
 		return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
 	}
@@ -84,10 +91,22 @@ func Parse(rawuri string) (*URI, error) {
 }
 
 func (u *URI) Raw() bool {
-	return u.Scheme == "bzzr"
+	return u.Scheme == "bzz-raw"
 }
 
 func (u *URI) Immutable() bool {
+	return u.Scheme == "bzz-immutable"
+}
+
+func (u *URI) List() bool {
+	return u.Scheme == "bzz-list"
+}
+
+func (u *URI) DeprecatedRaw() bool {
+	return u.Scheme == "bzzr"
+}
+
+func (u *URI) DeprecatedImmutable() bool {
 	return u.Scheme == "bzzi"
 }
 

swarm/api/uri_test.go

@@ -23,11 +23,14 @@ import (
 
 func TestParseURI(t *testing.T) {
 	type test struct {
-		uri             string
+		uri                       string
-		expectURI       *URI
+		expectURI                 *URI
-		expectErr       bool
+		expectErr                 bool
-		expectRaw       bool
+		expectRaw                 bool
-		expectImmutable bool
+		expectImmutable           bool
+		expectList                bool
+		expectDeprecatedRaw       bool
+		expectDeprecatedImmutable bool
 	}
 	tests := []test{
 		{
@@ -47,13 +50,13 @@ func TestParseURI(t *testing.T) {
 			expectURI: &URI{Scheme: "bzz"},
 		},
 		{
-			uri:             "bzzi:",
+			uri:             "bzz-immutable:",
-			expectURI:       &URI{Scheme: "bzzi"},
+			expectURI:       &URI{Scheme: "bzz-immutable"},
 			expectImmutable: true,
 		},
 		{
-			uri:       "bzzr:",
+			uri:       "bzz-raw:",
-			expectURI: &URI{Scheme: "bzzr"},
+			expectURI: &URI{Scheme: "bzz-raw"},
 			expectRaw: true,
 		},
 		{
@@ -69,18 +72,18 @@ func TestParseURI(t *testing.T) {
 			expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
 		},
 		{
-			uri:       "bzzr:/",
+			uri:       "bzz-raw:/",
-			expectURI: &URI{Scheme: "bzzr"},
+			expectURI: &URI{Scheme: "bzz-raw"},
 			expectRaw: true,
 		},
 		{
-			uri:       "bzzr:/abc123",
+			uri:       "bzz-raw:/abc123",
-			expectURI: &URI{Scheme: "bzzr", Addr: "abc123"},
+			expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123"},
 			expectRaw: true,
 		},
 		{
-			uri:       "bzzr:/abc123/path/to/entry",
+			uri:       "bzz-raw:/abc123/path/to/entry",
-			expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"},
+			expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123", Path: "path/to/entry"},
 			expectRaw: true,
 		},
 		{
@@ -95,6 +98,36 @@ func TestParseURI(t *testing.T) {
 			uri:       "bzz://abc123/path/to/entry",
 			expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
 		},
+		{
+			uri:        "bzz-list:",
+			expectURI:  &URI{Scheme: "bzz-list"},
+			expectList: true,
+		},
+		{
+			uri:        "bzz-list:/",
+			expectURI:  &URI{Scheme: "bzz-list"},
+			expectList: true,
+		},
+		{
+			uri:                 "bzzr:",
+			expectURI:           &URI{Scheme: "bzzr"},
+			expectDeprecatedRaw: true,
+		},
+		{
+			uri:                 "bzzr:/",
+			expectURI:           &URI{Scheme: "bzzr"},
+			expectDeprecatedRaw: true,
+		},
+		{
+			uri:                       "bzzi:",
+			expectURI:                 &URI{Scheme: "bzzi"},
+			expectDeprecatedImmutable: true,
+		},
+		{
+			uri:                       "bzzi:/",
+			expectURI:                 &URI{Scheme: "bzzi"},
+			expectDeprecatedImmutable: true,
+		},
 	}
 	for _, x := range tests {
 		actual, err := Parse(x.uri)
@@ -116,5 +149,14 @@ func TestParseURI(t *testing.T) {
 		if actual.Immutable() != x.expectImmutable {
 			t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
 		}
+		if actual.List() != x.expectList {
+			t.Fatalf("expected %s list to be %t, got %t", x.uri, x.expectList, actual.List())
+		}
+		if actual.DeprecatedRaw() != x.expectDeprecatedRaw {
+			t.Fatalf("expected %s deprecated raw to be %t, got %t", x.uri, x.expectDeprecatedRaw, actual.DeprecatedRaw())
+		}
+		if actual.DeprecatedImmutable() != x.expectDeprecatedImmutable {
+			t.Fatalf("expected %s deprecated immutable to be %t, got %t", x.uri, x.expectDeprecatedImmutable, actual.DeprecatedImmutable())
+		}
 	}
 }

swarm/dev/scripts/random-uploads.sh

@@ -46,7 +46,7 @@ main() {
 }
 
 do_random_upload() {
-  curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzzr:/"
+  curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzz-raw:/"
 }
 
 random_data() {

whisper/whisperv6/peer.go

@@ -153,21 +153,26 @@ func (peer *Peer) expire() {
 // broadcast iterates over the collection of envelopes and transmits yet unknown
 // ones over the network.
 func (p *Peer) broadcast() error {
-	var cnt int
 	envelopes := p.host.Envelopes()
+	bundle := make([]*Envelope, 0, len(envelopes))
 	for _, envelope := range envelopes {
 		if !p.marked(envelope) && envelope.PoW() >= p.powRequirement {
-			err := p2p.Send(p.ws, messagesCode, envelope)
+			bundle = append(bundle, envelope)
-			if err != nil {
-				return err
-			} else {
-				p.mark(envelope)
-				cnt++
-			}
 		}
 	}
-	if cnt > 0 {
+
-		log.Trace("broadcast", "num. messages", cnt)
+	if len(bundle) > 0 {
+		// transmit the batch of envelopes
+		if err := p2p.Send(p.ws, messagesCode, bundle); err != nil {
+			return err
+		}
+
+		// mark envelopes only if they were successfully sent
+		for _, e := range bundle {
+			p.mark(e)
+		}
+
+		log.Trace("broadcast", "num. messages", len(bundle))
 	}
 	return nil
 }

whisper/whisperv6/whisper.go

@@ -557,18 +557,26 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
 			log.Warn("unxepected status message received", "peer", p.peer.ID())
 		case messagesCode:
 			// decode the contained envelopes
-			var envelope Envelope
+			var envelopes []*Envelope
-			if err := packet.Decode(&envelope); err != nil {
+			if err := packet.Decode(&envelopes); err != nil {
-				log.Warn("failed to decode envelope, peer will be disconnected", "peer", p.peer.ID(), "err", err)
+				log.Warn("failed to decode envelopes, peer will be disconnected", "peer", p.peer.ID(), "err", err)
-				return errors.New("invalid envelope")
+				return errors.New("invalid envelopes")
 			}
-			cached, err := wh.add(&envelope)
+
-			if err != nil {
+			trouble := false
-				log.Warn("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err)
+			for _, env := range envelopes {
-				return errors.New("invalid envelope")
+				cached, err := wh.add(env)
+				if err != nil {
+					trouble = true
+					log.Error("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err)
+				}
+				if cached {
+					p.mark(env)
+				}
 			}
-			if cached {
+
-				p.mark(&envelope)
+			if trouble {
+				return errors.New("invalid envelope")
 			}
 		case powRequirementCode:
 			s := rlp.NewStream(packet.Payload, uint64(packet.Size))