whisper: message format refactoring, initial commit

whisper/whisperv6/doc.go

@@ -48,13 +48,13 @@ const (
 	p2pMessageCode       = 127 // peer-to-peer message (to be consumed by the peer, but not forwarded any further)
 	NumberOfMessageCodes = 128
 
-	paddingMask   = byte(3)
+	auxFieldSizeMask = byte(3) // mask used to extract the size of auxiliary field from the flags
 	signatureFlag    = byte(4)
 
 	TopicLength     = 4  // in bytes
 	signatureLength = 65 // in bytes
 	aesKeyLength    = 32 // in bytes
-	AESNonceLength  = 12 // in bytes
+	AESNonceLength  = 12 // in bytes; also returned by aesgcm.NonceSize()
 	keyIdSize       = 32 // in bytes
 	bloomFilterSize = 64 // in bytes
 
@@ -64,7 +64,7 @@ const (
 	DefaultMaxMessageSize = uint32(1024 * 1024)
 	DefaultMinimumPoW     = 0.2
 
-	padSizeLimit      = 256 // just an arbitrary number, could be changed without breaking the protocol (must not exceed 2^24)
+	padSizeLimit      = 256 // just an arbitrary number, could be changed without breaking the protocol
 	messageQueueLimit = 1024
 
 	expirationCycle   = time.Second

whisper/whisperv6/envelope.go

@@ -204,20 +204,23 @@ func (e *Envelope) Open(watcher *Filter) (msg *ReceivedMessage) {
 		return nil
 	}
 
+	var symmetric bool
 	if watcher.expectsAsymmetricEncryption() {
 		msg, _ = e.OpenAsymmetric(watcher.KeyAsym)
 		if msg != nil {
+			symmetric = false
 			msg.Dst = &watcher.KeyAsym.PublicKey
 		}
 	} else if watcher.expectsSymmetricEncryption() {
 		msg, _ = e.OpenSymmetric(watcher.KeySym)
 		if msg != nil {
+			symmetric = true
 			msg.SymKeyHash = crypto.Keccak256Hash(watcher.KeySym)
 		}
 	}
 
 	if msg != nil {
-		ok := msg.Validate()
+		ok := msg.ValidateAndParse(symmetric)
 		if !ok {
 			return nil
 		}

whisper/whisperv6/message.go

@@ -25,6 +25,7 @@ import (
 	crand "crypto/rand"
 	"encoding/binary"
 	"errors"
+	mrand "math/rand"
 	"strconv"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -89,80 +90,95 @@ func (msg *ReceivedMessage) isAsymmetricEncryption() bool {
 // NewMessage creates and initializes a non-signed, non-encrypted Whisper message.
 func NewSentMessage(params *MessageParams) (*sentMessage, error) {
 	msg := sentMessage{}
-	msg.Raw = make([]byte, 1, len(params.Payload)+len(params.Padding)+signatureLength+padSizeLimit)
+	msg.Raw = make([]byte, 1, 5+len(params.Payload)+len(params.Padding)+signatureLength+padSizeLimit)
 	msg.Raw[0] = 0 // set all the flags to zero
-	err := msg.appendPadding(params)
+	msg.addPayloadSizeField(params.Payload)
-	if err != nil {
-		return nil, err
-	}
 	msg.Raw = append(msg.Raw, params.Payload...)
-	return &msg, nil
+	err := msg.appendPadding(params)
+	return &msg, err
 }
 
-// getSizeOfLength returns the number of bytes necessary to encode the entire size padding (including these bytes)
+// appendPayloadSizeField appends the auxiliary field containing the size of payload
-func getSizeOfLength(b []byte) (sz int, err error) {
+func (msg *sentMessage) addPayloadSizeField(payload []byte) {
-	sz = intSize(len(b))      // first iteration
+	fieldSize := getAuxFieldSize(payload)
-	sz = intSize(len(b) + sz) // second iteration
+	field := make([]byte, fieldSize)
-	if sz > 3 {
+	binary.LittleEndian.PutUint32(field, uint32(len(payload)))
-		err = errors.New("oversized padding parameter")
+	msg.Raw = append(msg.Raw, field...)
-	}
+	msg.Raw[0] |= byte(fieldSize)
-	return sz, err
 }
 
-// sizeOfIntSize returns minimal number of bytes necessary to encode an integer value
+// getSizeOfLength returns the number of bytes necessary to encode the size of padding
-func intSize(i int) (s int) {
+//func getAuxFieldSize(payload []byte) (sz int, err error) {
-	for s = 1; i >= 256; s++ {
+//	sz = intSize(len(b))      // first iteration
-		i /= 256
+//	sz = intSize(len(b) + sz) // second iteration
+//	if sz > 3 {
+//		err = errors.New("oversized padding parameter")
+//	}
+//	return sz, err
+//}
+
+// getAuxFieldSize returns the number of bytes necessary to encode the size of payload
+func getAuxFieldSize(payload []byte) int {
+	s := 1
+	for i := len(payload); i >= 256; i /= 256 {
+		s++
 	}
 	return s
 }
 
-// appendPadding appends the pseudorandom padding bytes and sets the padding flag.
+// appendPadding appends the padding specified in params.
-// The last byte contains the size of padding (thus, its size must not exceed 256).
+// If no padding is provided in params, then random padding is generated.
 func (msg *sentMessage) appendPadding(params *MessageParams) error {
-	rawSize := len(params.Payload) + 1
+	if len(params.Padding) != 0 {
+		// padding data was provided by the Dapp, just use it as is
+		msg.Raw = append(msg.Raw, params.Padding...)
+		return nil
+	}
+
+	auxFieldSize := getAuxFieldSize(params.Payload)
+	rawSize := 1 + auxFieldSize + len(params.Payload)
 	if params.Src != nil {
 		rawSize += signatureLength
 	}
-
 	if params.KeySym != nil {
 		rawSize += AESNonceLength
 	}
 	odd := rawSize % padSizeLimit
 
-	if len(params.Padding) != 0 {
+	//if len(params.Padding) != 0 {
-		padSize := len(params.Padding)
+	//	// padding data was provided by the Dapp, just use it as is
-		padLengthSize, err := getSizeOfLength(params.Padding)
+	//	padSize := len(params.Padding)
+	//	padLengthSize, err := intSize(len(params.Padding))
+	//	if err != nil {
+	//		return err
+	//	}
+	//	totalPadSize := padSize + padLengthSize
+	//	buf := make([]byte, 8)
+	//	binary.LittleEndian.PutUint32(buf, uint32(totalPadSize))
+	//	buf = buf[:padLengthSize]
+	//	msg.Raw = append(msg.Raw, buf...)
+	//	msg.Raw = append(msg.Raw, params.Padding...)
+	//	msg.Raw[0] |= byte(padLengthSize) // number of bytes indicating the padding size
+	//} else if odd != 0 {
+	paddingSize := padSizeLimit - odd
+	//if totalPadSize > 255 {
+	//	// this algorithm is only valid if padSizeLimit < 256.
+	//	// if padSizeLimit will ever change, please fix the algorithm
+	//	// (please see also ReceivedMessage.extractPadding() function).
+	//	panic("please fix the padding algorithm before releasing new version")
+	//}
+	pad := make([]byte, paddingSize)
+	_, err := crand.Read(pad)
 	if err != nil {
 		return err
 	}
-		totalPadSize := padSize + padLengthSize
+	if !validateSymmetricKey(pad) {
-		buf := make([]byte, 8)
+		return errors.New("failed to generate random padding of size " + strconv.Itoa(paddingSize))
-		binary.LittleEndian.PutUint32(buf, uint32(totalPadSize))
-		buf = buf[:padLengthSize]
-		msg.Raw = append(msg.Raw, buf...)
-		msg.Raw = append(msg.Raw, params.Padding...)
-		msg.Raw[0] |= byte(padLengthSize) // number of bytes indicating the padding size
-	} else if odd != 0 {
-		totalPadSize := padSizeLimit - odd
-		if totalPadSize > 255 {
-			// this algorithm is only valid if padSizeLimit < 256.
-			// if padSizeLimit will ever change, please fix the algorithm
-			// (please see also ReceivedMessage.extractPadding() function).
-			panic("please fix the padding algorithm before releasing new version")
-		}
-		buf := make([]byte, totalPadSize)
-		_, err := crand.Read(buf[1:])
-		if err != nil {
-			return err
-		}
-		if totalPadSize > 6 && !validateSymmetricKey(buf) {
-			return errors.New("failed to generate random padding of size " + strconv.Itoa(totalPadSize))
-		}
-		buf[0] = byte(totalPadSize)
-		msg.Raw = append(msg.Raw, buf...)
-		msg.Raw[0] |= byte(0x1) // number of bytes indicating the padding size
 	}
+	//buf[0] = byte(totalPadSize)
+	msg.Raw = append(msg.Raw, pad...)
+	//msg.Raw[0] |= byte(0x1) // number of bytes indicating the padding size
+	//}
 	return nil
 }
 
@@ -175,14 +191,15 @@ func (msg *sentMessage) sign(key *ecdsa.PrivateKey) error {
 		return nil
 	}
 
-	msg.Raw[0] |= signatureFlag
+	msg.Raw[0] |= signatureFlag // it is important to set this flag before signing
 	hash := crypto.Keccak256(msg.Raw)
 	signature, err := crypto.Sign(hash, key)
 	if err != nil {
-		msg.Raw[0] &= ^signatureFlag // clear the flag
+		msg.Raw[0] ^= signatureFlag // clear the flag
 		return err
 	}
 	msg.Raw = append(msg.Raw, signature...)
+
 	return nil
 }
 
@@ -204,7 +221,6 @@ func (msg *sentMessage) encryptSymmetric(key []byte) (err error) {
 	if !validateSymmetricKey(key) {
 		return errors.New("invalid key provided for symmetric encryption")
 	}
-
 	block, err := aes.NewCipher(key)
 	if err != nil {
 		return err
@@ -213,18 +229,41 @@ func (msg *sentMessage) encryptSymmetric(key []byte) (err error) {
 	if err != nil {
 		return err
 	}
-
+	salt, err := generateSalt(aesgcm)
-	// never use more than 2^32 random nonces with a given key
-	salt := make([]byte, aesgcm.NonceSize())
-	_, err = crand.Read(salt)
 	if err != nil {
 		return err
-	} else if !validateSymmetricKey(salt) {
+	}
-		return errors.New("crypto/rand failed to generate salt")
+	encrypted := aesgcm.Seal(nil, salt, msg.Raw, nil)
+	msg.Raw = append(encrypted, salt...)
+	return nil
 }
 
-	msg.Raw = append(aesgcm.Seal(nil, salt, msg.Raw, nil), salt...)
+func generateSalt(aesgcm cipher.AEAD) ([]byte, error) {
-	return nil
+	// never use more than 2^32 random nonces with a given key
+	sz := aesgcm.NonceSize()
+	x1 := make([]byte, sz)
+	x2 := make([]byte, sz)
+	salt := make([]byte, sz)
+
+	_, err := crand.Read(x1)
+	if err != nil {
+		return nil, err
+	} else if !validateSymmetricKey(x1) {
+		return nil, errors.New("crypto/rand failed to generate salt")
+	}
+	_, err = mrand.Read(x2)
+	if err != nil {
+		return nil, err
+	} else if !validateSymmetricKey(x2) {
+		return nil, errors.New("math/rand failed to generate salt")
+	}
+	for i := 0; i < sz; i++ {
+		salt[i] = x1[i] ^ x2[i]
+	}
+	if !validateSymmetricKey(salt) {
+		return nil, errors.New("failed to generate salt")
+	}
+	return salt, nil
 }
 
 // Wrap bundles the message into an Envelope to transmit over the network.
@@ -295,31 +334,50 @@ func (msg *ReceivedMessage) decryptAsymmetric(key *ecdsa.PrivateKey) error {
 	return err
 }
 
-// Validate checks the validity and extracts the fields in case of success
+// Validate checks the message validity and extracts the fields in case of success.
-func (msg *ReceivedMessage) Validate() bool {
+func (msg *ReceivedMessage) ValidateAndParse(symmetric bool) bool {
 	end := len(msg.Raw)
 	if end < 1 {
 		return false
 	}
 
+	if symmetric {
+		end -= AESNonceLength
+	}
+
 	if isMessageSigned(msg.Raw[0]) {
 		end -= signatureLength
 		if end <= 1 {
 			return false
 		}
-		msg.Signature = msg.Raw[end:]
+		msg.Signature = msg.Raw[end : end+signatureLength]
 		msg.Src = msg.SigToPubKey()
 		if msg.Src == nil {
 			return false
 		}
 	}
 
-	padSize, ok := msg.extractPadding(end)
+	beg := 1
-	if !ok {
+	payloadSize := 0
+	auxFieldSize := int(msg.Raw[0] & auxFieldSizeMask) // number of bytes indicating the size of payload
+	if auxFieldSize != 0 {
+		payloadSize = int(bytesToUintLittleEndian(msg.Raw[beg : beg+auxFieldSize]))
+		if payloadSize+1 > end {
 			return false
 		}
+		beg += auxFieldSize
+		msg.Payload = msg.Raw[beg : beg+payloadSize]
+	}
+
+	beg += payloadSize
+	msg.Padding = msg.Raw[beg:end]
+
+	//padSize, ok := msg.extractPadding(end)
+	//if !ok {
+	//	return false
+	//}
+	//msg.Payload = msg.Raw[1+padSize : end]
 
-	msg.Payload = msg.Raw[1+padSize : end]
 	return true
 }
 
@@ -327,19 +385,18 @@ func (msg *ReceivedMessage) Validate() bool {
 // although we don't support sending messages with padding size
 // exceeding 255 bytes, such messages are perfectly valid, and
 // can be successfully decrypted.
-func (msg *ReceivedMessage) extractPadding(end int) (int, bool) {
+//func (msg *ReceivedMessage) extractPadding(end int) (int, bool) {
-	paddingSize := 0
+//	payloadSize := 0
-	sz := int(msg.Raw[0] & paddingMask) // number of bytes indicating the entire size of padding (including these bytes)
+//	auxFieldSize := int(msg.Raw[0] & auxFieldSizeMask) // number of bytes indicating the size of payload
-	// could be zero -- it means no padding
+//	if sz != 0 {
-	if sz != 0 {
+//		paddingSize = int(bytesToUintLittleEndian(msg.Raw[1 : 1+sz]))
-		paddingSize = int(bytesToUintLittleEndian(msg.Raw[1 : 1+sz]))
+//		if paddingSize < sz || paddingSize+1 > end {
-		if paddingSize < sz || paddingSize+1 > end {
+//			return 0, false
-			return 0, false
+//		}
-		}
+//		msg.Padding = msg.Raw[1+sz : 1+paddingSize]
-		msg.Padding = msg.Raw[1+sz : 1+paddingSize]
+//	}
-	}
+//	return paddingSize, true
-	return paddingSize, true
+//}
-}
 
 // Recover retrieves the public key of the message signer.
 func (msg *ReceivedMessage) SigToPubKey() *ecdsa.PublicKey {
@@ -353,7 +410,7 @@ func (msg *ReceivedMessage) SigToPubKey() *ecdsa.PublicKey {
 	return pub
 }
 
-// hash calculates the SHA3 checksum of the message flags, payload and padding.
+// hash calculates the SHA3 checksum of the message flags, auxiliary field, payload and padding.
 func (msg *ReceivedMessage) hash() []byte {
 	if isMessageSigned(msg.Raw[0]) {
 		sz := len(msg.Raw) - signatureLength

whisper/whisperv6/message_test.go

@@ -90,7 +90,7 @@ func singleMessageTest(t *testing.T, symmetric bool) {
 		t.Fatalf("failed to encrypt with seed %d: %s.", seed, err)
 	}
 
-	if !decrypted.Validate() {
+	if !decrypted.ValidateAndParse(symmetric) {
 		t.Fatalf("failed to validate with seed %d.", seed)
 	}