crypto/ecies: fix ECIES invalid-curve handling (#33669)

Fix ECIES invalid-curve handling in RLPx handshake (reject invalid
ephemeral pubkeys early)
- Add curve validation in crypto/ecies.GenerateShared to reject invalid
public keys before ECDH.
- Update RLPx PoC test to assert invalid curve points fail with
ErrInvalidPublicKey.
 
Motivation / Context
RLPx handshake uses ECIES decryption on unauthenticated network input.
Prior to this change, an invalid-curve ephemeral public key would
proceed into ECDH and only fail at MAC verification, returning
ErrInvalidMessage. This allows an oracle on decrypt success/failure and
leaves the code path vulnerable to invalid-curve/small-subgroup attacks.
The fix enforces IsOnCurve validation up front.

crypto/ecies/ecies.go

@@ -124,6 +124,9 @@ func (prv *PrivateKey) GenerateShared(pub *PublicKey, skLen, macLen int) (sk []b
 	if prv.PublicKey.Curve != pub.Curve {
 		return nil, ErrInvalidCurve
 	}
+	if pub.X == nil || pub.Y == nil || !pub.Curve.IsOnCurve(pub.X, pub.Y) {
+		return nil, ErrInvalidPublicKey
+	}
 	if skLen+macLen > MaxSharedKeyLength(pub) {
 		return nil, ErrSharedKeyTooBig
 	}

p2p/rlpx/rlpx_oracle_poc_test.go

@@ -0,0 +1,57 @@
+package rlpx
+
+import (
+	"bytes"
+	"errors"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/crypto/ecies"
+)
+
+func TestHandshakeECIESInvalidCurveOracle(t *testing.T) {
+	initKey, err := crypto.GenerateKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	respKey, err := crypto.GenerateKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	init := handshakeState{
+		initiator: true,
+		remote:    ecies.ImportECDSAPublic(&respKey.PublicKey),
+	}
+	authMsg, err := init.makeAuthMsg(initKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	packet, err := init.sealEIP8(authMsg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var recv handshakeState
+	if _, err := recv.readMsg(new(authMsgV4), respKey, bytes.NewReader(packet)); err != nil {
+		t.Fatalf("expected valid packet to decrypt: %v", err)
+	}
+
+	tampered := append([]byte(nil), packet...)
+	if len(tampered) < 2+65 {
+		t.Fatalf("unexpected packet length %d", len(tampered))
+	}
+	tampered[2] = 0x04
+	for i := 1; i < 65; i++ {
+		tampered[2+i] = 0x00
+	}
+
+	var recv2 handshakeState
+	_, err = recv2.readMsg(new(authMsgV4), respKey, bytes.NewReader(tampered))
+	if err == nil {
+		t.Fatal("expected decryption failure for invalid curve point")
+	}
+	if !errors.Is(err, ecies.ErrInvalidPublicKey) {
+		t.Fatalf("unexpected error: %v", err)
+	}
+}