From cd73f598441d37e0b136772b3a839a330ee4f6b1 Mon Sep 17 00:00:00 2001 From: Vlad Date: Tue, 9 Jan 2018 00:03:59 +0200 Subject: [PATCH] whisper: message format refactoring, initial commit --- whisper/whisperv6/doc.go | 8 +- whisper/whisperv6/envelope.go | 5 +- whisper/whisperv6/message.go | 229 +++++++++++++++++++----------- whisper/whisperv6/message_test.go | 2 +- 4 files changed, 152 insertions(+), 92 deletions(-) diff --git a/whisper/whisperv6/doc.go b/whisper/whisperv6/doc.go index da1b4ee5ba..eae2dcb77f 100644 --- a/whisper/whisperv6/doc.go +++ b/whisper/whisperv6/doc.go @@ -48,13 +48,13 @@ const ( p2pMessageCode = 127 // peer-to-peer message (to be consumed by the peer, but not forwarded any further) NumberOfMessageCodes = 128 - paddingMask = byte(3) - signatureFlag = byte(4) + auxFieldSizeMask = byte(3) // mask used to extract the size of auxiliary field from the flags + signatureFlag = byte(4) TopicLength = 4 // in bytes signatureLength = 65 // in bytes aesKeyLength = 32 // in bytes - AESNonceLength = 12 // in bytes + AESNonceLength = 12 // in bytes; also returned by aesgcm.NonceSize() keyIdSize = 32 // in bytes bloomFilterSize = 64 // in bytes @@ -64,7 +64,7 @@ const ( DefaultMaxMessageSize = uint32(1024 * 1024) DefaultMinimumPoW = 0.2 - padSizeLimit = 256 // just an arbitrary number, could be changed without breaking the protocol (must not exceed 2^24) + padSizeLimit = 256 // just an arbitrary number, could be changed without breaking the protocol messageQueueLimit = 1024 expirationCycle = time.Second diff --git a/whisper/whisperv6/envelope.go b/whisper/whisperv6/envelope.go index 9ed712b934..6cbefaef67 100644 --- a/whisper/whisperv6/envelope.go +++ b/whisper/whisperv6/envelope.go @@ -204,20 +204,23 @@ func (e *Envelope) Open(watcher *Filter) (msg *ReceivedMessage) { return nil } + var symmetric bool if watcher.expectsAsymmetricEncryption() { msg, _ = e.OpenAsymmetric(watcher.KeyAsym) if msg != nil { + symmetric = false msg.Dst = &watcher.KeyAsym.PublicKey } } else if watcher.expectsSymmetricEncryption() { msg, _ = e.OpenSymmetric(watcher.KeySym) if msg != nil { + symmetric = true msg.SymKeyHash = crypto.Keccak256Hash(watcher.KeySym) } } if msg != nil { - ok := msg.Validate() + ok := msg.ValidateAndParse(symmetric) if !ok { return nil } diff --git a/whisper/whisperv6/message.go b/whisper/whisperv6/message.go index f8df50336e..09425f098f 100644 --- a/whisper/whisperv6/message.go +++ b/whisper/whisperv6/message.go @@ -25,6 +25,7 @@ import ( crand "crypto/rand" "encoding/binary" "errors" + mrand "math/rand" "strconv" "github.com/ethereum/go-ethereum/common" @@ -89,80 +90,95 @@ func (msg *ReceivedMessage) isAsymmetricEncryption() bool { // NewMessage creates and initializes a non-signed, non-encrypted Whisper message. func NewSentMessage(params *MessageParams) (*sentMessage, error) { msg := sentMessage{} - msg.Raw = make([]byte, 1, len(params.Payload)+len(params.Padding)+signatureLength+padSizeLimit) + msg.Raw = make([]byte, 1, 5+len(params.Payload)+len(params.Padding)+signatureLength+padSizeLimit) msg.Raw[0] = 0 // set all the flags to zero - err := msg.appendPadding(params) - if err != nil { - return nil, err - } + msg.addPayloadSizeField(params.Payload) msg.Raw = append(msg.Raw, params.Payload...) - return &msg, nil + err := msg.appendPadding(params) + return &msg, err } -// getSizeOfLength returns the number of bytes necessary to encode the entire size padding (including these bytes) -func getSizeOfLength(b []byte) (sz int, err error) { - sz = intSize(len(b)) // first iteration - sz = intSize(len(b) + sz) // second iteration - if sz > 3 { - err = errors.New("oversized padding parameter") - } - return sz, err +// appendPayloadSizeField appends the auxiliary field containing the size of payload +func (msg *sentMessage) addPayloadSizeField(payload []byte) { + fieldSize := getAuxFieldSize(payload) + field := make([]byte, fieldSize) + binary.LittleEndian.PutUint32(field, uint32(len(payload))) + msg.Raw = append(msg.Raw, field...) + msg.Raw[0] |= byte(fieldSize) } -// sizeOfIntSize returns minimal number of bytes necessary to encode an integer value -func intSize(i int) (s int) { - for s = 1; i >= 256; s++ { - i /= 256 +// getSizeOfLength returns the number of bytes necessary to encode the size of padding +//func getAuxFieldSize(payload []byte) (sz int, err error) { +// sz = intSize(len(b)) // first iteration +// sz = intSize(len(b) + sz) // second iteration +// if sz > 3 { +// err = errors.New("oversized padding parameter") +// } +// return sz, err +//} + +// getAuxFieldSize returns the number of bytes necessary to encode the size of payload +func getAuxFieldSize(payload []byte) int { + s := 1 + for i := len(payload); i >= 256; i /= 256 { + s++ } return s } -// appendPadding appends the pseudorandom padding bytes and sets the padding flag. -// The last byte contains the size of padding (thus, its size must not exceed 256). +// appendPadding appends the padding specified in params. +// If no padding is provided in params, then random padding is generated. func (msg *sentMessage) appendPadding(params *MessageParams) error { - rawSize := len(params.Payload) + 1 + if len(params.Padding) != 0 { + // padding data was provided by the Dapp, just use it as is + msg.Raw = append(msg.Raw, params.Padding...) + return nil + } + + auxFieldSize := getAuxFieldSize(params.Payload) + rawSize := 1 + auxFieldSize + len(params.Payload) if params.Src != nil { rawSize += signatureLength } - if params.KeySym != nil { rawSize += AESNonceLength } odd := rawSize % padSizeLimit - if len(params.Padding) != 0 { - padSize := len(params.Padding) - padLengthSize, err := getSizeOfLength(params.Padding) - if err != nil { - return err - } - totalPadSize := padSize + padLengthSize - buf := make([]byte, 8) - binary.LittleEndian.PutUint32(buf, uint32(totalPadSize)) - buf = buf[:padLengthSize] - msg.Raw = append(msg.Raw, buf...) - msg.Raw = append(msg.Raw, params.Padding...) - msg.Raw[0] |= byte(padLengthSize) // number of bytes indicating the padding size - } else if odd != 0 { - totalPadSize := padSizeLimit - odd - if totalPadSize > 255 { - // this algorithm is only valid if padSizeLimit < 256. - // if padSizeLimit will ever change, please fix the algorithm - // (please see also ReceivedMessage.extractPadding() function). - panic("please fix the padding algorithm before releasing new version") - } - buf := make([]byte, totalPadSize) - _, err := crand.Read(buf[1:]) - if err != nil { - return err - } - if totalPadSize > 6 && !validateSymmetricKey(buf) { - return errors.New("failed to generate random padding of size " + strconv.Itoa(totalPadSize)) - } - buf[0] = byte(totalPadSize) - msg.Raw = append(msg.Raw, buf...) - msg.Raw[0] |= byte(0x1) // number of bytes indicating the padding size + //if len(params.Padding) != 0 { + // // padding data was provided by the Dapp, just use it as is + // padSize := len(params.Padding) + // padLengthSize, err := intSize(len(params.Padding)) + // if err != nil { + // return err + // } + // totalPadSize := padSize + padLengthSize + // buf := make([]byte, 8) + // binary.LittleEndian.PutUint32(buf, uint32(totalPadSize)) + // buf = buf[:padLengthSize] + // msg.Raw = append(msg.Raw, buf...) + // msg.Raw = append(msg.Raw, params.Padding...) + // msg.Raw[0] |= byte(padLengthSize) // number of bytes indicating the padding size + //} else if odd != 0 { + paddingSize := padSizeLimit - odd + //if totalPadSize > 255 { + // // this algorithm is only valid if padSizeLimit < 256. + // // if padSizeLimit will ever change, please fix the algorithm + // // (please see also ReceivedMessage.extractPadding() function). + // panic("please fix the padding algorithm before releasing new version") + //} + pad := make([]byte, paddingSize) + _, err := crand.Read(pad) + if err != nil { + return err } + if !validateSymmetricKey(pad) { + return errors.New("failed to generate random padding of size " + strconv.Itoa(paddingSize)) + } + //buf[0] = byte(totalPadSize) + msg.Raw = append(msg.Raw, pad...) + //msg.Raw[0] |= byte(0x1) // number of bytes indicating the padding size + //} return nil } @@ -175,14 +191,15 @@ func (msg *sentMessage) sign(key *ecdsa.PrivateKey) error { return nil } - msg.Raw[0] |= signatureFlag + msg.Raw[0] |= signatureFlag // it is important to set this flag before signing hash := crypto.Keccak256(msg.Raw) signature, err := crypto.Sign(hash, key) if err != nil { - msg.Raw[0] &= ^signatureFlag // clear the flag + msg.Raw[0] ^= signatureFlag // clear the flag return err } msg.Raw = append(msg.Raw, signature...) + return nil } @@ -204,7 +221,6 @@ func (msg *sentMessage) encryptSymmetric(key []byte) (err error) { if !validateSymmetricKey(key) { return errors.New("invalid key provided for symmetric encryption") } - block, err := aes.NewCipher(key) if err != nil { return err @@ -213,20 +229,43 @@ func (msg *sentMessage) encryptSymmetric(key []byte) (err error) { if err != nil { return err } - - // never use more than 2^32 random nonces with a given key - salt := make([]byte, aesgcm.NonceSize()) - _, err = crand.Read(salt) + salt, err := generateSalt(aesgcm) if err != nil { return err - } else if !validateSymmetricKey(salt) { - return errors.New("crypto/rand failed to generate salt") } - - msg.Raw = append(aesgcm.Seal(nil, salt, msg.Raw, nil), salt...) + encrypted := aesgcm.Seal(nil, salt, msg.Raw, nil) + msg.Raw = append(encrypted, salt...) return nil } +func generateSalt(aesgcm cipher.AEAD) ([]byte, error) { + // never use more than 2^32 random nonces with a given key + sz := aesgcm.NonceSize() + x1 := make([]byte, sz) + x2 := make([]byte, sz) + salt := make([]byte, sz) + + _, err := crand.Read(x1) + if err != nil { + return nil, err + } else if !validateSymmetricKey(x1) { + return nil, errors.New("crypto/rand failed to generate salt") + } + _, err = mrand.Read(x2) + if err != nil { + return nil, err + } else if !validateSymmetricKey(x2) { + return nil, errors.New("math/rand failed to generate salt") + } + for i := 0; i < sz; i++ { + salt[i] = x1[i] ^ x2[i] + } + if !validateSymmetricKey(salt) { + return nil, errors.New("failed to generate salt") + } + return salt, nil +} + // Wrap bundles the message into an Envelope to transmit over the network. func (msg *sentMessage) Wrap(options *MessageParams) (envelope *Envelope, err error) { if options.TTL == 0 { @@ -295,31 +334,50 @@ func (msg *ReceivedMessage) decryptAsymmetric(key *ecdsa.PrivateKey) error { return err } -// Validate checks the validity and extracts the fields in case of success -func (msg *ReceivedMessage) Validate() bool { +// Validate checks the message validity and extracts the fields in case of success. +func (msg *ReceivedMessage) ValidateAndParse(symmetric bool) bool { end := len(msg.Raw) if end < 1 { return false } + if symmetric { + end -= AESNonceLength + } + if isMessageSigned(msg.Raw[0]) { end -= signatureLength if end <= 1 { return false } - msg.Signature = msg.Raw[end:] + msg.Signature = msg.Raw[end : end+signatureLength] msg.Src = msg.SigToPubKey() if msg.Src == nil { return false } } - padSize, ok := msg.extractPadding(end) - if !ok { - return false + beg := 1 + payloadSize := 0 + auxFieldSize := int(msg.Raw[0] & auxFieldSizeMask) // number of bytes indicating the size of payload + if auxFieldSize != 0 { + payloadSize = int(bytesToUintLittleEndian(msg.Raw[beg : beg+auxFieldSize])) + if payloadSize+1 > end { + return false + } + beg += auxFieldSize + msg.Payload = msg.Raw[beg : beg+payloadSize] } - msg.Payload = msg.Raw[1+padSize : end] + beg += payloadSize + msg.Padding = msg.Raw[beg:end] + + //padSize, ok := msg.extractPadding(end) + //if !ok { + // return false + //} + //msg.Payload = msg.Raw[1+padSize : end] + return true } @@ -327,19 +385,18 @@ func (msg *ReceivedMessage) Validate() bool { // although we don't support sending messages with padding size // exceeding 255 bytes, such messages are perfectly valid, and // can be successfully decrypted. -func (msg *ReceivedMessage) extractPadding(end int) (int, bool) { - paddingSize := 0 - sz := int(msg.Raw[0] & paddingMask) // number of bytes indicating the entire size of padding (including these bytes) - // could be zero -- it means no padding - if sz != 0 { - paddingSize = int(bytesToUintLittleEndian(msg.Raw[1 : 1+sz])) - if paddingSize < sz || paddingSize+1 > end { - return 0, false - } - msg.Padding = msg.Raw[1+sz : 1+paddingSize] - } - return paddingSize, true -} +//func (msg *ReceivedMessage) extractPadding(end int) (int, bool) { +// payloadSize := 0 +// auxFieldSize := int(msg.Raw[0] & auxFieldSizeMask) // number of bytes indicating the size of payload +// if sz != 0 { +// paddingSize = int(bytesToUintLittleEndian(msg.Raw[1 : 1+sz])) +// if paddingSize < sz || paddingSize+1 > end { +// return 0, false +// } +// msg.Padding = msg.Raw[1+sz : 1+paddingSize] +// } +// return paddingSize, true +//} // Recover retrieves the public key of the message signer. func (msg *ReceivedMessage) SigToPubKey() *ecdsa.PublicKey { @@ -353,7 +410,7 @@ func (msg *ReceivedMessage) SigToPubKey() *ecdsa.PublicKey { return pub } -// hash calculates the SHA3 checksum of the message flags, payload and padding. +// hash calculates the SHA3 checksum of the message flags, auxiliary field, payload and padding. func (msg *ReceivedMessage) hash() []byte { if isMessageSigned(msg.Raw[0]) { sz := len(msg.Raw) - signatureLength diff --git a/whisper/whisperv6/message_test.go b/whisper/whisperv6/message_test.go index c90bcc01ed..d0305c2e60 100644 --- a/whisper/whisperv6/message_test.go +++ b/whisper/whisperv6/message_test.go @@ -90,7 +90,7 @@ func singleMessageTest(t *testing.T, symmetric bool) { t.Fatalf("failed to encrypt with seed %d: %s.", seed, err) } - if !decrypted.Validate() { + if !decrypted.ValidateAndParse(symmetric) { t.Fatalf("failed to validate with seed %d.", seed) }