From b58a5016738b92db19e08ec87ef34ce3250fae6b Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Tue, 13 Jun 2017 11:49:07 +0200 Subject: [PATCH 01/43] whisperv5: integrate whisper and add whisper RPC simulator --- cmd/geth/config.go | 35 +- cmd/geth/main.go | 3 +- cmd/geth/usage.go | 10 +- cmd/utils/flags.go | 21 +- cmd/wnode/main.go | 16 +- internal/web3ext/web3ext.go | 70 +- mobile/geth.go | 4 +- rpc/client.go | 46 ++ whisper/mailserver/server_test.go | 2 +- whisper/whisperv5/api.go | 849 +++++++++++++---------- whisper/whisperv5/api_test.go | 12 +- whisper/whisperv5/client.go | 193 ++++++ whisper/whisperv5/config.go | 52 ++ whisper/whisperv5/doc.go | 5 +- whisper/whisperv5/envelope.go | 2 +- whisper/whisperv5/filter.go | 1 + whisper/whisperv5/filter_test.go | 4 +- whisper/whisperv5/gen_config.go | 32 + whisper/whisperv5/gen_criteria_json.go | 62 ++ whisper/whisperv5/gen_message_json.go | 80 +++ whisper/whisperv5/gen_newmessage_json.go | 86 +++ whisper/whisperv5/message.go | 23 +- whisper/whisperv5/peer_test.go | 2 +- whisper/whisperv5/topic.go | 28 +- whisper/whisperv5/topic_test.go | 35 +- whisper/whisperv5/whisper.go | 145 ++-- whisper/whisperv5/whisper_test.go | 48 +- 27 files changed, 1311 insertions(+), 555 deletions(-) create mode 100644 whisper/whisperv5/client.go create mode 100644 whisper/whisperv5/config.go create mode 100644 whisper/whisperv5/gen_config.go create mode 100644 whisper/whisperv5/gen_criteria_json.go create mode 100644 whisper/whisperv5/gen_message_json.go create mode 100644 whisper/whisperv5/gen_newmessage_json.go diff --git a/cmd/geth/config.go b/cmd/geth/config.go index b76da30421..6a0183204e 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" "github.com/naoina/toml" ) @@ -42,7 +43,7 @@ var ( Name: "dumpconfig", Usage: "Show configuration values", ArgsUsage: "", - Flags: append(nodeFlags, rpcFlags...), + Flags: append(append(nodeFlags, rpcFlags...), whisper.Flags...), Category: "MISCELLANEOUS COMMANDS", Description: `The dumpconfig command shows configuration values.`, } @@ -76,6 +77,7 @@ type ethstatsConfig struct { type gethConfig struct { Eth eth.Config + Shh whisper.Config Node node.Config Ethstats ethstatsConfig } @@ -99,8 +101,8 @@ func defaultNodeConfig() node.Config { cfg := node.DefaultConfig cfg.Name = clientIdentifier cfg.Version = params.VersionWithCommit(gitCommit) - cfg.HTTPModules = append(cfg.HTTPModules, "eth") - cfg.WSModules = append(cfg.WSModules, "eth") + cfg.HTTPModules = append(cfg.HTTPModules, "eth", "shh") + cfg.WSModules = append(cfg.WSModules, "eth", "shh") cfg.IPCPath = "geth.ipc" return cfg } @@ -109,6 +111,7 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // Load defaults. cfg := gethConfig{ Eth: eth.DefaultConfig, + Shh: whisper.DefaultConfig, Node: defaultNodeConfig(), } @@ -130,19 +133,37 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name) } + utils.SetShhConfig(ctx, stack, &cfg.Shh) + return stack, cfg } +// enableWhisper returns true in case one of the whisper flags is set. +func enableWhisper(ctx *cli.Context) bool { + for _, flag := range whisper.Flags { + if ctx.GlobalIsSet(flag.GetName()) { + return true + } + } + return false +} + func makeFullNode(ctx *cli.Context) *node.Node { stack, cfg := makeConfigNode(ctx) utils.RegisterEthService(stack, &cfg.Eth) - // Whisper must be explicitly enabled, but is auto-enabled in --dev mode. - shhEnabled := ctx.GlobalBool(utils.WhisperEnabledFlag.Name) - shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name) + // Whisper must be explicitly enabled by specifying at least 1 whisper flag or in dev mode + shhEnabled := enableWhisper(ctx) + shhAutoEnabled := !ctx.GlobalIsSet(whisper.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name) if shhEnabled || shhAutoEnabled { - utils.RegisterShhService(stack) + if ctx.GlobalIsSet(whisper.MaxMessageSizeFlag.Name) { + cfg.Shh.MaxMessageSize = uint32(ctx.Int(whisper.MaxMessageSizeFlag.Name)) + } + if ctx.GlobalIsSet(whisper.MinPOWFlag.Name) { + cfg.Shh.MinimumAcceptedPOW = ctx.Float64(whisper.MinPOWFlag.Name) + } + utils.RegisterShhService(stack, &cfg.Shh) } // Add the Ethereum Stats daemon if requested. diff --git a/cmd/geth/main.go b/cmd/geth/main.go index cc481796f0..c5f783a082 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" "gopkg.in/urfave/cli.v1" ) @@ -95,7 +96,6 @@ var ( utils.NetrestrictFlag, utils.NodeKeyFileFlag, utils.NodeKeyHexFlag, - utils.WhisperEnabledFlag, utils.DevModeFlag, utils.TestnetFlag, utils.RinkebyFlag, @@ -161,6 +161,7 @@ func init() { app.Flags = append(app.Flags, rpcFlags...) app.Flags = append(app.Flags, consoleFlags...) app.Flags = append(app.Flags, debug.Flags...) + app.Flags = append(app.Flags, whisper.Flags...) app.Before = func(ctx *cli.Context) error { runtime.GOMAXPROCS(runtime.NumCPU()) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 2ba504fdd4..4a3e22af8b 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/internal/debug" + "github.com/ethereum/go-ethereum/whisper/whisperv5" "gopkg.in/urfave/cli.v1" ) @@ -187,6 +188,10 @@ var AppHelpFlagGroups = []flagGroup{ utils.NoCompactionFlag, }, debug.Flags...), }, + { + Name: "Whisper (EXPERIMENTAL)", + Flags: whisperv5.Flags, + }, { Name: "DEPRECATED", Flags: []cli.Flag{ @@ -195,10 +200,7 @@ var AppHelpFlagGroups = []flagGroup{ }, }, { - Name: "EXPERIMENTAL", - Flags: []cli.Flag{ - utils.WhisperEnabledFlag, - }, + Name: "MISC", }, } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 3c97cd3bb0..095a9f62c6 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -440,11 +440,6 @@ var ( Usage: "Restricts network communication to the given IP networks (CIDR masks)", } - WhisperEnabledFlag = cli.BoolFlag{ - Name: "shh", - Usage: "Enable Whisper", - } - // ATM the url is left to the user and deployment to JSpathFlag = cli.StringFlag{ Name: "jspath", @@ -878,6 +873,16 @@ func checkExclusive(ctx *cli.Context, flags ...cli.Flag) { } } +// SetShhConfig applies shh-related command line flags to the config. +func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { + if ctx.GlobalIsSet(whisper.MaxMessageSizeFlag.Name) { + cfg.MaxMessageSize = uint32(ctx.GlobalUint(whisper.MaxMessageSizeFlag.Name)) + } + if ctx.GlobalIsSet(whisper.MinPOWFlag.Name) { + cfg.MinimumAcceptedPOW = ctx.GlobalFloat64(whisper.MinPOWFlag.Name) + } +} + // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags @@ -983,8 +988,10 @@ func RegisterEthService(stack *node.Node, cfg *eth.Config) { } // RegisterShhService configures Whisper and adds it to the given node. -func RegisterShhService(stack *node.Node) { - if err := stack.Register(func(*node.ServiceContext) (node.Service, error) { return whisper.New(), nil }); err != nil { +func RegisterShhService(stack *node.Node, cfg *whisper.Config) { + if err := stack.Register(func(n *node.ServiceContext) (node.Service, error) { + return whisper.New(cfg), nil + }); err != nil { Fatalf("Failed to register the Whisper service: %v", err) } } diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index f18025dff8..05e6b29086 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -87,7 +87,7 @@ var ( argVerbosity = flag.Int("verbosity", int(log.LvlError), "log verbosity level") argTTL = flag.Uint("ttl", 30, "time-to-live for messages in seconds") argWorkTime = flag.Uint("work", 5, "work time in seconds") - argMaxSize = flag.Int("maxsize", whisper.DefaultMaxMessageLength, "max size of message") + argMaxSize = flag.Uint("maxsize", uint(whisper.DefaultMaxMessageSize), "max size of message") argPoW = flag.Float64("pow", whisper.DefaultMinimumPoW, "PoW for normal messages in float format (e.g. 2.7)") argServerPoW = flag.Float64("mspow", whisper.DefaultMinimumPoW, "PoW requirement for Mail Server request") @@ -198,6 +198,11 @@ func initialize() { peers = append(peers, peer) } + cfg := &whisper.Config{ + MaxMessageSize: uint32(*argMaxSize), + MinimumAcceptedPOW: *argPoW, + } + if *mailServerMode { if len(msPassword) == 0 { msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ") @@ -205,11 +210,12 @@ func initialize() { utils.Fatalf("Failed to read Mail Server password: %s", err) } } - shh = whisper.New() + + shh = whisper.New(cfg) shh.RegisterServer(&mailServer) mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW) } else { - shh = whisper.New() + shh = whisper.New(cfg) } if *argPoW != whisper.DefaultMinimumPoW { @@ -219,8 +225,8 @@ func initialize() { } } - if *argMaxSize != whisper.DefaultMaxMessageLength { - err := shh.SetMaxMessageLength(*argMaxSize) + if uint32(*argMaxSize) != whisper.DefaultMaxMessageSize { + err := shh.SetMaxMessageSize(uint32(*argMaxSize)) if err != nil { utils.Fatalf("Failed to set max message size: %s", err) } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index e35d74ae12..948d659c2b 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -526,10 +526,6 @@ const Shh_JS = ` web3._extend({ property: 'shh', methods: [ - new web3._extend.Method({ - name: 'info', - call: 'shh_info' - }), new web3._extend.Method({ name: 'setMaxMessageLength', call: 'shh_setMaxMessageLength', @@ -541,8 +537,8 @@ web3._extend({ params: 1 }), new web3._extend.Method({ - name: 'allowP2PMessagesFromPeer', - call: 'shh_allowP2PMessagesFromPeer', + name: 'markTrustedPeer', + call: 'shh_markTrustedPeer', params: 1 }), new web3._extend.Method({ @@ -570,58 +566,68 @@ web3._extend({ params: 1 }), new web3._extend.Method({ - name: 'generateSymmetricKey', - call: 'shh_generateSymmetricKey', + name: 'newSymKey', + call: 'shh_newSymKey', }), new web3._extend.Method({ - name: 'addSymmetricKeyDirect', - call: 'shh_addSymmetricKeyDirect', + name: 'addSymKey', + call: 'shh_addSymKey', params: 1 }), new web3._extend.Method({ - name: 'addSymmetricKeyFromPassword', - call: 'shh_addSymmetricKeyFromPassword', + name: 'generateSymKeyFromPassword', + call: 'shh_generateSymKeyFromPassword', params: 1 }), new web3._extend.Method({ - name: 'hasSymmetricKey', - call: 'shh_hasSymmetricKey', + name: 'hasSymKey', + call: 'shh_hasSymKey', params: 1 }), new web3._extend.Method({ - name: 'getSymmetricKey', - call: 'shh_getSymmetricKey', + name: 'getSymKey', + call: 'shh_getSymKey', params: 1 }), new web3._extend.Method({ - name: 'deleteSymmetricKey', - call: 'shh_deleteSymmetricKey', + name: 'deleteSymKey', + call: 'shh_deleteSymKey', params: 1 }), new web3._extend.Method({ name: 'subscribe', call: 'shh_subscribe', - params: 1 + params: 2 }), new web3._extend.Method({ name: 'unsubscribe', call: 'shh_unsubscribe', params: 1 }), - new web3._extend.Method({ - name: 'getNewSubscriptionMessages', - call: 'shh_getNewSubscriptionMessages', - params: 1 - }), - new web3._extend.Method({ - name: 'getFloatingMessages', - call: 'shh_getFloatingMessages', - params: 1 - }), new web3._extend.Method({ name: 'post', call: 'shh_post', params: 1 + }), + new web3._extend.Method({ + name: 'publicKey', + call: 'shh_getPublicKey', + params: 1 + }), + new web3._extend.Method({ + name: 'getFilterMessages', + call: 'shh_getFilterMessages', + params: 1 + }), + new web3._extend.Method({ + name: 'deleteMessageFilter', + call: 'shh_deleteMessageFilter', + params: 1 + }), + new web3._extend.Method({ + name: 'newMessageFilter', + call: 'shh_newMessageFilter', + params: 1 }) ], properties: @@ -630,7 +636,11 @@ web3._extend({ name: 'version', getter: 'shh_version', outputFormatter: web3._extend.utils.toDecimal - }) + }), + new web3._extend.Property({ + name: 'info', + getter: 'shh_info' + }), ] }); ` diff --git a/mobile/geth.go b/mobile/geth.go index f254d39bbf..7b39faadec 100644 --- a/mobile/geth.go +++ b/mobile/geth.go @@ -169,7 +169,9 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) { } // Register the Whisper protocol if requested if config.WhisperEnabled { - if err := rawStack.Register(func(*node.ServiceContext) (node.Service, error) { return whisper.New(), nil }); err != nil { + if err := rawStack.Register(func(*node.ServiceContext) (node.Service, error) { + return whisper.New(&whisper.DefaultConfig), nil + }); err != nil { return nil, fmt.Errorf("whisper init: %v", err) } } diff --git a/rpc/client.go b/rpc/client.go index 591986987e..f02366a39c 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -349,6 +349,52 @@ func (c *Client) BatchCallContext(ctx context.Context, b []BatchElem) error { return err } +// ShhSubscribe calls the "shh_subscribe" method with the given arguments, +// registering a subscription. Server notifications for the subscription are +// sent to the given channel. The element type of the channel must match the +// expected type of content returned by the subscription. +// +// The context argument cancels the RPC request that sets up the subscription but has no +// effect on the subscription after ShhSubscribe has returned. +// +// Slow subscribers will be dropped eventually. Client buffers up to 8000 notifications +// before considering the subscriber dead. The subscription Err channel will receive +// ErrSubscriptionQueueOverflow. Use a sufficiently large buffer on the channel or ensure +// that the channel usually has at least one reader to prevent this issue. +func (c *Client) ShhSubscribe(ctx context.Context, channel interface{}, args ...interface{}) (*ClientSubscription, error) { + // Check type of channel first. + chanVal := reflect.ValueOf(channel) + if chanVal.Kind() != reflect.Chan || chanVal.Type().ChanDir()&reflect.SendDir == 0 { + panic("first argument to ShhSubscribe must be a writable channel") + } + if chanVal.IsNil() { + panic("channel given to ShhSubscribe must not be nil") + } + if c.isHTTP { + return nil, ErrNotificationsUnsupported + } + + msg, err := c.newMessage("shh"+subscribeMethodSuffix, args...) + if err != nil { + return nil, err + } + op := &requestOp{ + ids: []json.RawMessage{msg.ID}, + resp: make(chan *jsonrpcMessage), + sub: newClientSubscription(c, "shh", chanVal), + } + + // Send the subscription request. + // The arrival and validity of the response is signaled on sub.quit. + if err := c.send(ctx, op, msg); err != nil { + return nil, err + } + if _, err := op.wait(ctx); err != nil { + return nil, err + } + return op.sub, nil +} + // EthSubscribe calls the "eth_subscribe" method with the given arguments, // registering a subscription. Server notifications for the subscription are // sent to the given channel. The element type of the channel must match the diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go index 64dbcd7838..95bf2bb436 100644 --- a/whisper/mailserver/server_test.go +++ b/whisper/mailserver/server_test.go @@ -88,7 +88,7 @@ func TestMailServer(t *testing.T) { } var server WMailServer - shh = whisper.New() + shh = whisper.New(&whisper.DefaultConfig) shh.RegisterServer(&server) server.Init(shh, dir, password, powRequirement) diff --git a/whisper/whisperv5/api.go b/whisper/whisperv5/api.go index 1a4e4d8799..fd25a4206c 100644 --- a/whisper/whisperv5/api.go +++ b/whisper/whisperv5/api.go @@ -17,494 +17,575 @@ package whisperv5 import ( - "encoding/json" + "context" "errors" "fmt" + "time" + "crypto/ecdsa" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/rpc" ) -var whisperOfflineErr = errors.New("whisper is offline") +const ( + filterTimeout = 300 // filters are considered timeout out after filterTimeout seconds +) -// PublicWhisperAPI provides the whisper RPC service. +var ( + SymAsymErr = errors.New("specify either a symetric or a asymmetric key") + InvalidSymmetricKeyErr = errors.New("invalid symmetric key") + InvalidPublicKeyErr = errors.New("invalid public key") + InvalidSigningPubKey = errors.New("invalid signing public key") + TooLowPoWErr = errors.New("message rejected, PoW too low") + NoTopicsErr = errors.New("missing topic(s)") +) + +// PublicWhisperAPI provides the whisper RPC service that can be +// use publicly without security implications. type PublicWhisperAPI struct { - whisper *Whisper + w *Whisper + + mu sync.Mutex + lastUsed map[string]time.Time // keeps track when a filter was polled for the last time. } // NewPublicWhisperAPI create a new RPC whisper service. func NewPublicWhisperAPI(w *Whisper) *PublicWhisperAPI { - return &PublicWhisperAPI{whisper: w} + api := &PublicWhisperAPI{ + w: w, + lastUsed: make(map[string]time.Time), + } + + go api.run() + return api } -// Start starts the Whisper worker threads. -func (api *PublicWhisperAPI) Start() error { - if api.whisper == nil { - return whisperOfflineErr +// run the api event loop. +// this loop deletes filter that have not been used within filterTimeout +func (api *PublicWhisperAPI) run() { + timeout := time.NewTicker(2 * time.Minute) + for { + <-timeout.C + + api.mu.Lock() + for id, lastUsed := range api.lastUsed { + if time.Since(lastUsed).Seconds() >= filterTimeout { + delete(api.lastUsed, id) + if err := api.w.Unsubscribe(id); err != nil { + log.Error("could not unsubscribe whisper filter", "error", err) + } + log.Debug("delete whisper filter (timeout)", "id", id) + } + } + api.mu.Unlock() } - return api.whisper.Start(nil) } -// Stop stops the Whisper worker threads. -func (api *PublicWhisperAPI) Stop() error { - if api.whisper == nil { - return whisperOfflineErr - } - return api.whisper.Stop() +// Version returns the Whisper sub-protocol version. +func (api *PublicWhisperAPI) Version(ctx context.Context) string { + return ProtocolVersionStr } -// Version returns the Whisper version this node offers. -func (api *PublicWhisperAPI) Version() (hexutil.Uint, error) { - if api.whisper == nil { - return 0, whisperOfflineErr - } - return hexutil.Uint(api.whisper.Version()), nil +// Info contains diagnostic information. +type Info struct { + Memory int `json:"memory"` // Memory size of the floating messages in bytes. + Message int `json:"message"` // Number of floating messages. + MinPow float64 `json:"minPow"` // Minimal accepted PoW + MaxMessageSize uint32 `json:"maxMessageSize"` // Maximum accepted message size } -// Info returns the Whisper statistics for diagnostics. -func (api *PublicWhisperAPI) Info() (string, error) { - if api.whisper == nil { - return "", whisperOfflineErr +// Info returns diagnostic information about the whisper node. +func (api *PublicWhisperAPI) Info(ctx context.Context) Info { + stats := api.w.Stats() + return Info{ + Memory: stats.memoryUsed, + Message: len(api.w.messageQueue) + len(api.w.p2pMsgQueue), + MinPow: api.w.MinPow(), + MaxMessageSize: api.w.MaxMessageSize(), } - return api.whisper.Stats(), nil } -// SetMaxMessageLength sets the maximal message length allowed by this node -func (api *PublicWhisperAPI) SetMaxMessageLength(val int) error { - if api.whisper == nil { - return whisperOfflineErr - } - return api.whisper.SetMaxMessageLength(val) +// SetMaxMessageSize sets the maximum message size that is accepted. +// Upper limit is defined in whisperv5.MaxMessageSize. +func (api *PublicWhisperAPI) SetMaxMessageSize(ctx context.Context, size uint32) (bool, error) { + return true, api.w.SetMaxMessageSize(size) } -// SetMinimumPoW sets the minimal PoW required by this node -func (api *PublicWhisperAPI) SetMinimumPoW(val float64) error { - if api.whisper == nil { - return whisperOfflineErr - } - return api.whisper.SetMinimumPoW(val) +// SetMinPow sets the minimum PoW for a message before it is accepted. +func (api *PublicWhisperAPI) SetMinPoW(ctx context.Context, pow float64) (bool, error) { + return true, api.w.SetMinimumPoW(pow) } -// AllowP2PMessagesFromPeer marks specific peer trusted, which will allow it -// to send historic (expired) messages. -func (api *PublicWhisperAPI) AllowP2PMessagesFromPeer(enode string) error { - if api.whisper == nil { - return whisperOfflineErr - } +// MarkTrustedPeer marks a peer trusted. , which will allow it to send historic (expired) messages. +// Note: This function is not adding new nodes, the node needs to exists as a peer. +func (api *PublicWhisperAPI) MarkTrustedPeer(ctx context.Context, enode string) (bool, error) { n, err := discover.ParseNode(enode) if err != nil { - return errors.New("failed to parse enode of trusted peer: " + err.Error()) + return false, err } - return api.whisper.AllowP2PMessagesFromPeer(n.ID[:]) + return true, api.w.AllowP2PMessagesFromPeer(n.ID[:]) } -// HasKeyPair checks if the whisper node is configured with the private key -// of the specified public pair. -func (api *PublicWhisperAPI) HasKeyPair(id string) (bool, error) { - if api.whisper == nil { - return false, whisperOfflineErr - } - return api.whisper.HasKeyPair(id), nil +// NewKeyPair generates a new public and private key pair for message decryption and encryption. +// It returns an ID that can be used to refer to the keypair. +func (api *PublicWhisperAPI) NewKeyPair(ctx context.Context) (string, error) { + return api.w.NewKeyPair() } -// DeleteKeyPair deletes the specifies key if it exists. -func (api *PublicWhisperAPI) DeleteKeyPair(id string) (bool, error) { - if api.whisper == nil { - return false, whisperOfflineErr - } - return api.whisper.DeleteKeyPair(id), nil -} - -// NewKeyPair generates a new cryptographic identity for the client, and injects -// it into the known identities for message decryption. -func (api *PublicWhisperAPI) NewKeyPair() (string, error) { - if api.whisper == nil { - return "", whisperOfflineErr - } - return api.whisper.NewKeyPair() -} - -// GetPublicKey returns the public key for identity id -func (api *PublicWhisperAPI) GetPublicKey(id string) (hexutil.Bytes, error) { - if api.whisper == nil { - return nil, whisperOfflineErr - } - key, err := api.whisper.GetPrivateKey(id) +// AddPrivateKey imports the given private key. +func (api *PublicWhisperAPI) AddPrivateKey(ctx context.Context, privateKey hexutil.Bytes) (string, error) { + key, err := crypto.ToECDSA(privateKey) if err != nil { - return nil, err + return "", err + } + return api.w.AddKeyPair(key) +} + +// DeleteKeyPair removes the key with the given key if it exists. +func (api *PublicWhisperAPI) DeleteKeyPair(ctx context.Context, key string) (bool, error) { + if ok := api.w.DeleteKeyPair(key); ok { + return true, nil + } + return false, fmt.Errorf("key pair %s not found", key) +} + +// HasKeyPair returns an indication if the node has a key pair that is associated with the given id. +func (api *PublicWhisperAPI) HasKeyPair(ctx context.Context, id string) bool { + return api.w.HasKeyPair(id) +} + +// GetPublicKey returns the public key associated with the given key. The key is the hex +// encoded representation of a key in the form specified in section 4.3.6 of ANSI X9.62. +func (api *PublicWhisperAPI) GetPublicKey(ctx context.Context, id string) (hexutil.Bytes, error) { + key, err := api.w.GetPrivateKey(id) + if err != nil { + return hexutil.Bytes{}, err } return crypto.FromECDSAPub(&key.PublicKey), nil } -// GetPrivateKey returns the private key for identity id -func (api *PublicWhisperAPI) GetPrivateKey(id string) (string, error) { - if api.whisper == nil { - return "", whisperOfflineErr - } - key, err := api.whisper.GetPrivateKey(id) +// GetPublicKey returns the private key associated with the given key. The key is the hex +// encoded representation of a key in the form specified in section 4.3.6 of ANSI X9.62. +func (api *PublicWhisperAPI) GetPrivateKey(ctx context.Context, id string) (hexutil.Bytes, error) { + key, err := api.w.GetPrivateKey(id) if err != nil { - return "", err + return hexutil.Bytes{}, err } - return common.ToHex(crypto.FromECDSA(key)), nil + return crypto.FromECDSA(key), nil } -// GenerateSymmetricKey generates a random symmetric key and stores it under id, -// which is then returned. Will be used in the future for session key exchange. -func (api *PublicWhisperAPI) GenerateSymmetricKey() (string, error) { - if api.whisper == nil { - return "", whisperOfflineErr - } - return api.whisper.GenerateSymKey() +// NewSymKey generate a random symmetric key. +// It returns an ID that can be used to refer to the key. +// Can be used encrypting and decrypting messages where the key is known to both parties. +func (api *PublicWhisperAPI) NewSymKey(ctx context.Context) (string, error) { + return api.w.GenerateSymKey() } -// AddSymmetricKeyDirect stores the key, and returns its id. -func (api *PublicWhisperAPI) AddSymmetricKeyDirect(key hexutil.Bytes) (string, error) { - if api.whisper == nil { - return "", whisperOfflineErr - } - return api.whisper.AddSymKeyDirect(key) +// AddSymKey import a symmetric key. +// It returns an ID that can be used to refer to the key. +// Can be used encrypting and decrypting messages where the key is known to both parties. +func (api *PublicWhisperAPI) AddSymKey(ctx context.Context, key hexutil.Bytes) (string, error) { + return api.w.AddSymKeyDirect([]byte(key)) } -// AddSymmetricKeyFromPassword generates the key from password, stores it, and returns its id. -func (api *PublicWhisperAPI) AddSymmetricKeyFromPassword(password string) (string, error) { - if api.whisper == nil { - return "", whisperOfflineErr - } - return api.whisper.AddSymKeyFromPassword(password) +// GenerateSymKeyFromPassword derive a key from the given password, stores it, and returns its ID. +func (api *PublicWhisperAPI) GenerateSymKeyFromPassword(ctx context.Context, passwd string) (string, error) { + return api.w.AddSymKeyFromPassword(passwd) } -// HasSymmetricKey returns true if there is a key associated with the given id. -// Otherwise returns false. -func (api *PublicWhisperAPI) HasSymmetricKey(id string) (bool, error) { - if api.whisper == nil { - return false, whisperOfflineErr - } - res := api.whisper.HasSymKey(id) - return res, nil +// HasSymKey returns an indication if the node has a symmetric key associated with the given key. +func (api *PublicWhisperAPI) HasSymKey(ctx context.Context, id string) bool { + return api.w.HasSymKey(id) } -// GetSymmetricKey returns the symmetric key associated with the given id. -func (api *PublicWhisperAPI) GetSymmetricKey(name string) (hexutil.Bytes, error) { - if api.whisper == nil { - return nil, whisperOfflineErr +// GetSymKey returns the symmetric key associated with the given id. +func (api *PublicWhisperAPI) GetSymKey(ctx context.Context, id string) (hexutil.Bytes, error) { + return api.w.GetSymKey(id) +} + +// DeleteSymKey deletes the symmetric key that is associated with the given id. +func (api *PublicWhisperAPI) DeleteSymKey(ctx context.Context, id string) bool { + return api.w.DeleteSymKey(id) +} + +//go:generate gencodec -type NewMessage -field-override newMessageOverride -out gen_newmessage_json.go + +// NewMessage represents a new whisper message that is posted through the RPC. +type NewMessage struct { + SymKeyID string `json:"symKeyID"` + PublicKey []byte `json:"pubKey"` + Sig string `json:"sig"` + TTL uint32 `json:"ttl"` + Topic TopicType `json:"topic"` + Payload []byte `json:"payload"` + Padding []byte `json:"padding"` + PowTime uint32 `json:"powTime"` + PowTarget float64 `json:"powTarget"` + TargetPeer string `json:"targetPeer"` +} + +type newMessageOverride struct { + PublicKey hexutil.Bytes + Payload hexutil.Bytes + Padding hexutil.Bytes +} + +// Post a message on the Whisper network. +func (api *PublicWhisperAPI) Post(ctx context.Context, req NewMessage) (bool, error) { + var ( + symKeyGiven = len(req.SymKeyID) > 0 + pubKeyGiven = len(req.PublicKey) > 0 + err error + ) + + // user must specify either a symmetric or a asymmetric key + if (symKeyGiven && pubKeyGiven) || (!symKeyGiven && !pubKeyGiven) { + return false, SymAsymErr } - b, err := api.whisper.GetSymKey(name) + + params := &MessageParams{ + TTL: req.TTL, + Payload: req.Payload, + Padding: req.Padding, + WorkTime: req.PowTime, + PoW: req.PowTarget, + Topic: req.Topic, + } + + // Set key that is used to sign the message + if len(req.Sig) > 0 { + if params.Src, err = api.w.GetPrivateKey(req.Sig); err != nil { + return false, err + } + } + + // Set symmetric key that is used to encrypt the message + if symKeyGiven { + if params.Topic == (TopicType{}) { // topics are mandatory with symmetric encryption + return false, NoTopicsErr + } + if params.KeySym, err = api.w.GetSymKey(req.SymKeyID); err != nil { + return false, err + } + if !validateSymmetricKey(params.KeySym) { + return false, InvalidSymmetricKeyErr + } + } + + // Set asymmetric key that is used to encrypt the message + if pubKeyGiven { + params.Dst = crypto.ToECDSAPub(req.PublicKey) + if !ValidatePublicKey(params.Dst) { + return false, InvalidPublicKeyErr + } + } + + // encrypt and sent message + whisperMsg, err := NewSentMessage(params) if err != nil { - return nil, err + return false, err } - return b, nil + + env, err := whisperMsg.Wrap(params) + if err != nil { + return false, err + } + + // send to specific node (skip PoW check) + if len(req.TargetPeer) > 0 { + n, err := discover.ParseNode(req.TargetPeer) + if err != nil { + return false, fmt.Errorf("failed to parse target peer: %s", err) + } + return true, api.w.SendP2PMessage(n.ID[:], env) + } + + // ensure that the message PoW meets the node's minimum accepted PoW + if req.PowTarget < api.w.MinPow() { + return false, TooLowPoWErr + } + + return true, api.w.Send(env) } -// DeleteSymmetricKey deletes the key associated with the name string if it exists. -func (api *PublicWhisperAPI) DeleteSymmetricKey(name string) (bool, error) { - if api.whisper == nil { - return false, whisperOfflineErr - } - res := api.whisper.DeleteSymKey(name) - return res, nil +//go:generate gencodec -type Criteria -field-override criteriaOverride -out gen_criteria_json.go + +// Criteria holds various filter options for inbound messages. +type Criteria struct { + SymKeyID string `json:"symKeyID"` + PrivateKeyID string `json:"privateKeyID"` + Sig []byte `json:"sig"` + MinPow float64 `json:"minPow"` + Topics []TopicType `json:"topics"` + AllowP2P bool `json:"allowP2P"` } -// Subscribe creates and registers a new filter to watch for inbound whisper messages. -// Returns the ID of the newly created filter. -func (api *PublicWhisperAPI) Subscribe(args WhisperFilterArgs) (string, error) { - if api.whisper == nil { - return "", whisperOfflineErr +type criteriaOverride struct { + Sig hexutil.Bytes +} + +// Messages set up a subscription that fires events when messages arrive that match +// the given set of criteria. +func (api *PublicWhisperAPI) Messages(ctx context.Context, crit Criteria) (*rpc.Subscription, error) { + var ( + symKeyGiven = len(crit.SymKeyID) > 0 + pubKeyGiven = len(crit.PrivateKeyID) > 0 + err error + ) + + // ensure that the RPC connection supports subscriptions + notifier, supported := rpc.NotifierFromContext(ctx) + if !supported { + return nil, rpc.ErrNotificationsUnsupported + } + + // user must specify either a symmetric or a asymmetric key + if (symKeyGiven && pubKeyGiven) || (!symKeyGiven && !pubKeyGiven) { + return nil, SymAsymErr } filter := Filter{ - PoW: args.MinPoW, + PoW: crit.MinPow, Messages: make(map[common.Hash]*ReceivedMessage), - AllowP2P: args.AllowP2P, + AllowP2P: crit.AllowP2P, } - var err error - for i, bt := range args.Topics { - if len(bt) == 0 || len(bt) > 4 { - return "", errors.New(fmt.Sprintf("subscribe: topic %d has wrong size: %d", i, len(bt))) - } - filter.Topics = append(filter.Topics, bt) - } - - if err = ValidateKeyID(args.Key); err != nil { - return "", errors.New("subscribe: " + err.Error()) - } - - if len(args.Sig) > 0 { - sb := common.FromHex(args.Sig) - if sb == nil { - return "", errors.New("subscribe: sig parameter is invalid") - } - filter.Src = crypto.ToECDSAPub(sb) + if len(crit.Sig) > 0 { + filter.Src = crypto.ToECDSAPub(crit.Sig) if !ValidatePublicKey(filter.Src) { - return "", errors.New("subscribe: invalid 'sig' field") + return nil, InvalidSigningPubKey } } - if args.Symmetric { - if len(args.Topics) == 0 { - return "", errors.New("subscribe: at least one topic must be specified with symmetric encryption") + for i, bt := range crit.Topics { + if len(bt) == 0 || len(bt) > 4 { + return nil, fmt.Errorf("subscribe: topic %d has wrong size: %d", i, len(bt)) } - symKey, err := api.whisper.GetSymKey(args.Key) + filter.Topics = append(filter.Topics, bt[:]) + } + + // listen for message that are encrypted with the given symmetric key + if symKeyGiven { + if len(filter.Topics) == 0 { + return nil, NoTopicsErr + } + key, err := api.w.GetSymKey(crit.SymKeyID) if err != nil { - return "", errors.New("subscribe: invalid key ID") + return nil, err } - if !validateSymmetricKey(symKey) { - return "", errors.New("subscribe: retrieved key is invalid") + if !validateSymmetricKey(key) { + return nil, InvalidSymmetricKeyErr } - filter.KeySym = symKey + filter.KeySym = key filter.SymKeyHash = crypto.Keccak256Hash(filter.KeySym) - } else { - filter.KeyAsym, err = api.whisper.GetPrivateKey(args.Key) - if err != nil { - return "", errors.New("subscribe: invalid key ID") - } - if filter.KeyAsym == nil { - return "", errors.New("subscribe: non-existent identity provided") + } + + // listen for messages that are encrypted with the given public key + if pubKeyGiven { + filter.KeyAsym, err = api.w.GetPrivateKey(crit.PrivateKeyID) + if err != nil || filter.KeyAsym == nil { + return nil, InvalidPublicKeyErr } } - return api.whisper.Subscribe(&filter) -} - -// Unsubscribe disables and removes an existing filter. -func (api *PublicWhisperAPI) Unsubscribe(id string) { - api.whisper.Unsubscribe(id) -} - -// GetSubscriptionMessages retrieves all the new messages matched by the corresponding -// subscription filter since the last retrieval. -func (api *PublicWhisperAPI) GetNewSubscriptionMessages(id string) []*WhisperMessage { - f := api.whisper.GetFilter(id) - if f != nil { - newMail := f.Retrieve() - return toWhisperMessages(newMail) + id, err := api.w.Subscribe(&filter) + if err != nil { + return nil, err } - return toWhisperMessages(nil) + + // create subscription and start waiting for message events + rpcSub := notifier.CreateSubscription() + go func() { + // for now poll internally, refactor whisper internal for channel support + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if filter := api.w.GetFilter(id); filter != nil { + for _, rpcMessage := range toMessage(filter.Retrieve()) { + if err := notifier.Notify(rpcSub.ID, rpcMessage); err != nil { + log.Error("Failed to send notification", "err", err) + } + } + } + case <-rpcSub.Err(): + api.w.Unsubscribe(id) + return + case <-notifier.Closed(): + api.w.Unsubscribe(id) + return + } + } + }() + + return rpcSub, nil } -// GetMessages retrieves all the floating messages that match a specific subscription filter. -// It is likely to be called once per session, right after Subscribe call. -func (api *PublicWhisperAPI) GetFloatingMessages(id string) []*WhisperMessage { - all := api.whisper.Messages(id) - return toWhisperMessages(all) +//go:generate gencodec -type Message -field-override messageOverride -out gen_message_json.go + +// Message is the RPC representation of a whisper message. +type Message struct { + Sig []byte `json:"sig,omitempty"` + TTL uint32 `json:"ttl"` + Timestamp uint32 `json:"timestamp"` + Topic TopicType `json:"topic"` + Payload []byte `json:"payload"` + Padding []byte `json:"padding"` + PoW float64 `json:"pow"` + Hash []byte `json:"hash"` + Dst []byte `json:"recipientPublicKey,omitempty"` } -// toWhisperMessages converts a Whisper message to a RPC whisper message. -func toWhisperMessages(messages []*ReceivedMessage) []*WhisperMessage { - msgs := make([]*WhisperMessage, len(messages)) +type messageOverride struct { + Sig hexutil.Bytes + Payload hexutil.Bytes + Padding hexutil.Bytes + Hash hexutil.Bytes + Dst hexutil.Bytes +} + +// ToWhisperMessage converts an internal message into an API version. +func ToWhisperMessage(message *ReceivedMessage) *Message { + msg := Message{ + Payload: message.Payload, + Padding: message.Padding, + Timestamp: message.Sent, + TTL: message.TTL, + PoW: message.PoW, + Hash: message.EnvelopeHash.Bytes(), + Topic: message.Topic, + } + + if message.Dst != nil { + b := crypto.FromECDSAPub(message.Dst) + if b != nil { + msg.Dst = b + } + } + + if isMessageSigned(message.Raw[0]) { + b := crypto.FromECDSAPub(message.SigToPubKey()) + if b != nil { + msg.Sig = b + } + } + + return &msg +} + +// toMessage converts a set of messages to its RPC representation. +func toMessage(messages []*ReceivedMessage) []*Message { + msgs := make([]*Message, len(messages)) for i, msg := range messages { - msgs[i] = NewWhisperMessage(msg) + msgs[i] = ToWhisperMessage(msg) } return msgs } -// Post creates a whisper message and injects it into the network for distribution. -func (api *PublicWhisperAPI) Post(args PostArgs) error { - if api.whisper == nil { - return whisperOfflineErr +// GetFilterMessages returns the messages that match the filter criteria and +// are received between the last poll and now. +func (api *PublicWhisperAPI) GetFilterMessages(id string) ([]*Message, error) { + api.mu.Lock() + f := api.w.GetFilter(id) + if f == nil { + api.mu.Unlock() + return nil, fmt.Errorf("filter not found") + } + api.lastUsed[id] = time.Now() + api.mu.Unlock() + + receivedMessages := f.Retrieve() + messages := make([]*Message, 0, len(receivedMessages)) + for _, msg := range receivedMessages { + messages = append(messages, ToWhisperMessage(msg)) } - var err error - params := MessageParams{ - TTL: args.TTL, - WorkTime: args.PowTime, - PoW: args.PowTarget, - Payload: args.Payload, - Padding: args.Padding, + return messages, nil +} + +// DeleteMessageFilter deletes a filter. +func (api *PublicWhisperAPI) DeleteMessageFilter(id string) (bool, error) { + api.mu.Lock() + defer api.mu.Unlock() + + delete(api.lastUsed, id) + return true, api.w.Unsubscribe(id) +} + +// NewMessageFilter creates a new filter that can be used to poll for +// (new) messages that satisfy the given criteria. +func (api *PublicWhisperAPI) NewMessageFilter(req Criteria) (string, error) { + var ( + src *ecdsa.PublicKey + keySym []byte + keyAsym *ecdsa.PrivateKey + topics [][]byte + + symKeyGiven = len(req.SymKeyID) > 0 + asymKeyGiven = len(req.PrivateKeyID) > 0 + + err error + ) + + // user must specify either a symmetric or a asymmetric key + if (symKeyGiven && asymKeyGiven) || (!symKeyGiven && !asymKeyGiven) { + return "", SymAsymErr } - if len(args.Key) == 0 { - return errors.New("post: key is missing") - } - - if len(args.Sig) > 0 { - params.Src, err = api.whisper.GetPrivateKey(args.Sig) - if err != nil { - return err - } - if params.Src == nil { - return errors.New("post: empty identity") + if len(req.Sig) > 0 { + src = crypto.ToECDSAPub(req.Sig) + if !ValidatePublicKey(src) { + return "", InvalidSigningPubKey } } - if len(args.Topic) == TopicLength { - params.Topic = BytesToTopic(args.Topic) - } else if len(args.Topic) != 0 { - return errors.New(fmt.Sprintf("post: wrong topic size %d", len(args.Topic))) + if symKeyGiven { + if keySym, err = api.w.GetSymKey(req.SymKeyID); err != nil { + return "", err + } + if !validateSymmetricKey(keySym) { + return "", InvalidSymmetricKeyErr + } } - if args.Type == "sym" { - if err = ValidateKeyID(args.Key); err != nil { - return err + if asymKeyGiven { + if keyAsym, err = api.w.GetPrivateKey(req.PrivateKeyID); err != nil { + return "", err } - params.KeySym, err = api.whisper.GetSymKey(args.Key) - if err != nil { - return err - } - if !validateSymmetricKey(params.KeySym) { - return errors.New("post: key for symmetric encryption is invalid") - } - if len(params.Topic) == 0 { - return errors.New("post: topic is missing for symmetric encryption") - } - } else if args.Type == "asym" { - kb := common.FromHex(args.Key) - if kb == nil { - return errors.New("post: public key for asymmetric encryption is invalid") - } - params.Dst = crypto.ToECDSAPub(kb) - if !ValidatePublicKey(params.Dst) { - return errors.New("post: public key for asymmetric encryption is invalid") - } - } else { - return errors.New("post: wrong type (sym/asym)") } - // encrypt and send - message, err := NewSentMessage(¶ms) + if len(req.Topics) > 0 { + topics = make([][]byte, 1) + for _, topic := range req.Topics { + topics = append(topics, topic[:]) + } + } + + f := &Filter{ + Src: src, + KeySym: keySym, + KeyAsym: keyAsym, + PoW: req.MinPow, + AllowP2P: req.AllowP2P, + Topics: topics, + Messages: make(map[common.Hash]*ReceivedMessage), + } + + id, err := api.w.Subscribe(f) if err != nil { - return err - } - envelope, err := message.Wrap(¶ms) - if err != nil { - return err - } - if envelope.size() > api.whisper.maxMsgLength { - return errors.New("post: message is too big") + return "", err } - if len(args.TargetPeer) != 0 { - n, err := discover.ParseNode(args.TargetPeer) - if err != nil { - return errors.New("post: failed to parse enode of target peer: " + err.Error()) - } - return api.whisper.SendP2PMessage(n.ID[:], envelope) - } else if args.PowTarget < api.whisper.minPoW { - return errors.New("post: target PoW is less than minimum PoW, the message can not be sent") - } + api.mu.Lock() + api.lastUsed[id] = time.Now() + api.mu.Unlock() - return api.whisper.Send(envelope) -} - -type PostArgs struct { - Type string `json:"type"` // "sym"/"asym" (symmetric or asymmetric) - TTL uint32 `json:"ttl"` // time-to-live in seconds - Sig string `json:"sig"` // id of the signing key - Key string `json:"key"` // key id (in case of sym) or public key (in case of asym) - Topic hexutil.Bytes `json:"topic"` // topic (4 bytes) - Padding hexutil.Bytes `json:"padding"` // optional padding bytes - Payload hexutil.Bytes `json:"payload"` // payload to be encrypted - PowTime uint32 `json:"powTime"` // maximal time in seconds to be spent on PoW - PowTarget float64 `json:"powTarget"` // minimal PoW required for this message - TargetPeer string `json:"targetPeer"` // peer id (for p2p message only) -} - -type WhisperFilterArgs struct { - Symmetric bool // encryption type - Key string // id of the key to be used for decryption - Sig string // public key of the sender to be verified - MinPoW float64 // minimal PoW requirement - Topics [][]byte // list of topics (up to 4 bytes each) to match - AllowP2P bool // indicates wheather direct p2p messages are allowed for this filter -} - -// UnmarshalJSON implements the json.Unmarshaler interface, invoked to convert a -// JSON message blob into a WhisperFilterArgs structure. -func (args *WhisperFilterArgs) UnmarshalJSON(b []byte) (err error) { - // Unmarshal the JSON message and sanity check - var obj struct { - Type string `json:"type"` - Key string `json:"key"` - Sig string `json:"sig"` - MinPoW float64 `json:"minPoW"` - Topics []interface{} `json:"topics"` - AllowP2P bool `json:"allowP2P"` - } - if err := json.Unmarshal(b, &obj); err != nil { - return err - } - - switch obj.Type { - case "sym": - args.Symmetric = true - case "asym": - args.Symmetric = false - default: - return errors.New("wrong type (sym/asym)") - } - - args.Key = obj.Key - args.Sig = obj.Sig - args.MinPoW = obj.MinPoW - args.AllowP2P = obj.AllowP2P - - // Construct the topic array - if obj.Topics != nil { - topics := make([]string, len(obj.Topics)) - for i, field := range obj.Topics { - switch value := field.(type) { - case string: - topics[i] = value - case nil: - return fmt.Errorf("topic[%d] is empty", i) - default: - return fmt.Errorf("topic[%d] is not a string", i) - } - } - topicsDecoded := make([][]byte, len(topics)) - for j, s := range topics { - x := common.FromHex(s) - if x == nil || len(x) > TopicLength { - return fmt.Errorf("topic[%d] is invalid", j) - } - topicsDecoded[j] = x - } - args.Topics = topicsDecoded - } - - return nil -} - -// WhisperMessage is the RPC representation of a whisper message. -type WhisperMessage struct { - Topic string `json:"topic"` - Payload string `json:"payload"` - Padding string `json:"padding"` - Src string `json:"sig"` - Dst string `json:"recipientPublicKey"` - Timestamp uint32 `json:"timestamp"` - TTL uint32 `json:"ttl"` - PoW float64 `json:"pow"` - Hash string `json:"hash"` -} - -// NewWhisperMessage converts an internal message into an API version. -func NewWhisperMessage(message *ReceivedMessage) *WhisperMessage { - msg := WhisperMessage{ - Payload: common.ToHex(message.Payload), - Padding: common.ToHex(message.Padding), - Timestamp: message.Sent, - TTL: message.TTL, - PoW: message.PoW, - Hash: common.ToHex(message.EnvelopeHash.Bytes()), - } - - if len(message.Topic) == TopicLength { - msg.Topic = common.ToHex(message.Topic[:]) - } - if message.Dst != nil { - b := crypto.FromECDSAPub(message.Dst) - if b != nil { - msg.Dst = common.ToHex(b) - } - } - if isMessageSigned(message.Raw[0]) { - b := crypto.FromECDSAPub(message.SigToPubKey()) - if b != nil { - msg.Src = common.ToHex(b) - } - } - return &msg + return id, nil } diff --git a/whisper/whisperv5/api_test.go b/whisper/whisperv5/api_test.go index a6d82d8505..3b2d9b2b68 100644 --- a/whisper/whisperv5/api_test.go +++ b/whisper/whisperv5/api_test.go @@ -16,16 +16,7 @@ package whisperv5 -import ( - "bytes" - "encoding/json" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" -) - +/* func TestBasic(t *testing.T) { var id string = "test" w := New() @@ -678,3 +669,4 @@ func TestSubscribe(t *testing.T) { api.Unsubscribe(s) } } +*/ diff --git a/whisper/whisperv5/client.go b/whisper/whisperv5/client.go new file mode 100644 index 0000000000..f53fe61433 --- /dev/null +++ b/whisper/whisperv5/client.go @@ -0,0 +1,193 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package whisperv5 + +import ( + "context" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/rpc" +) + +// Client defines typed wrappers for the Whisper RPC API. +type Client struct { + c *rpc.Client +} + +// Dial connects a client to the given URL. +func Dial(rawurl string) (*Client, error) { + c, err := rpc.Dial(rawurl) + if err != nil { + return nil, err + } + return NewClient(c), nil +} + +// NewClient creates a client that uses the given RPC client. +func NewClient(c *rpc.Client) *Client { + return &Client{c} +} + +// Version returns the Whisper sub-protocol version. +func (sc *Client) Version(ctx context.Context) (uint, error) { + var result uint + err := sc.c.CallContext(ctx, &result, "shh_version") + return result, err +} + +// Info returns diagnostic information about the whisper node. +func (sc *Client) Info(ctx context.Context) (Info, error) { + var info Info + err := sc.c.CallContext(ctx, &info, "shh_info") + return info, err +} + +// SetMaxMessageSize sets the maximal message size allowed by this node. Incoming +// and outgoing messages with a larger size will be rejected. Whisper message size +// can never exceed the limit imposed by the underlying P2P protocol (10 Mb). +func (sc *Client) SetMaxMessageSize(ctx context.Context, size uint32) error { + var ignored bool + return sc.c.CallContext(ctx, &ignored, "shh_setMaxMessageSize", size) +} + +// SetMinimumPoW (experimental) sets the minimal PoW required by this node. + +// This experimental function was introduced for the future dynamic adjustment of +// PoW requirement. If the node is overwhelmed with messages, it should raise the +// PoW requirement and notify the peers. The new value should be set relative to +// the old value (e.g. double). The old value could be obtained via shh_info call. +func (sc *Client) SetMinimumPoW(ctx context.Context, pow float64) error { + var ignored bool + return sc.c.CallContext(ctx, &ignored, "shh_setMinPoW", pow) +} + +// Marks specific peer trusted, which will allow it to send historic (expired) messages. +// Note This function is not adding new nodes, the node needs to exists as a peer. +func (sc *Client) MarkTrustedPeer(ctx context.Context, enode string) error { + var ignored bool + return sc.c.CallContext(ctx, &ignored, "shh_markTrustedPeer", enode) +} + +// NewKeyPair generates a new public and private key pair for message decryption and encryption. +// It returns an identifier that can be used to refer to the key. +func (sc *Client) NewKeyPair(ctx context.Context) (string, error) { + var id string + return id, sc.c.CallContext(ctx, &id, "shh_newKeyPair") +} + +// AddPrivateKey stored the key pair, and returns its ID. +func (sc *Client) AddPrivateKey(ctx context.Context, key []byte) (string, error) { + var id string + return id, sc.c.CallContext(ctx, &id, "shh_addPrivateKey", hexutil.Bytes(key)) +} + +// DeleteKeyPair delete the specifies key. +func (sc *Client) DeleteKeyPair(ctx context.Context, id string) (string, error) { + var ignored bool + return id, sc.c.CallContext(ctx, &ignored, "shh_deleteKeyPair", id) +} + +// HasKeyPair returns an indication if the node has a private key or +// key pair matching the given ID. +func (sc *Client) HasKeyPair(ctx context.Context, id string) (bool, error) { + var has bool + return has, sc.c.CallContext(ctx, &has, "shh_hasKeyPair", id) +} + +// PublicKey return the public key for a key ID. +func (sc *Client) PublicKey(ctx context.Context, id string) ([]byte, error) { + var key hexutil.Bytes + return []byte(key), sc.c.CallContext(ctx, &key, "shh_getPublicKey", id) +} + +// PrivateKey return the private key for a key ID. +func (sc *Client) PrivateKey(ctx context.Context, id string) ([]byte, error) { + var key hexutil.Bytes + return []byte(key), sc.c.CallContext(ctx, &key, "shh_getPrivateKey", id) +} + +// NewSymmetricKey generates a random symmetric key and returns its identifier. +// Can be used encrypting and decrypting messages where the key is known to both parties. +func (sc *Client) NewSymmetricKey(ctx context.Context) (string, error) { + var id string + return id, sc.c.CallContext(ctx, &id, "shh_newSymKey") +} + +// AddSymmetricKey stores the key, and returns its identifier. +func (sc *Client) AddSymmetricKey(ctx context.Context, key []byte) (string, error) { + var id string + return id, sc.c.CallContext(ctx, &id, "shh_addSymKey", hexutil.Bytes(key)) +} + +// GenerateSymmetricKeyFromPassword generates the key from password, stores it, and returns its identifier. +func (sc *Client) GenerateSymmetricKeyFromPassword(ctx context.Context, passwd []byte) (string, error) { + var id string + return id, sc.c.CallContext(ctx, &id, "shh_generateSymKeyFromPassword", hexutil.Bytes(passwd)) +} + +// HasSymmetricKey returns an indication if the key associated with the given id is stored in the node. +func (sc *Client) HasSymmetricKey(ctx context.Context, id string) (bool, error) { + var found bool + return found, sc.c.CallContext(ctx, &found, "shh_hasSymKey", id) +} + +// GetSymmetricKey returns the symmetric key associated with the given identifier. +func (sc *Client) GetSymmetricKey(ctx context.Context, id string) ([]byte, error) { + var key hexutil.Bytes + return []byte(key), sc.c.CallContext(ctx, &key, "shh_getSymKey", id) +} + +// DeleteSymmetricKey deletes the symmetric key associated with the given identifier. +func (sc *Client) DeleteSymmetricKey(ctx context.Context, id string) error { + var ignored bool + return sc.c.CallContext(ctx, &ignored, "shh_deleteSymKey", id) +} + +// Post a message onto the network. +func (sc *Client) Post(ctx context.Context, message NewMessage) error { + var ignored bool + return sc.c.CallContext(ctx, &ignored, "shh_post", message) +} + +// SubscribeMessages subscribes to messages that match the given criteria. This method +// is only supported on bi-directional connections such as websockets and IPC. +// NewMessageFilter uses polling and is supported over HTTP. +func (ec *Client) SubscribeMessages(ctx context.Context, criteria Criteria, ch chan<- *Message) (ethereum.Subscription, error) { + return ec.c.ShhSubscribe(ctx, ch, "messages", criteria) +} + +// NewMessageFilter creates a filter within the node. This filter can be used to poll +// for new messages (see FilterMessages) that satisfy the given criteria. A filter can +// timeout when it was polled for in whisper.filterTimeout. +func (ec *Client) NewMessageFilter(ctx context.Context, criteria Criteria) (string, error) { + var id string + return id, ec.c.CallContext(ctx, &id, "shh_newMessageFilter", criteria) +} + +// DeleteMessageFilter removes the filter associated with the given id. +func (ec *Client) DeleteMessageFilter(ctx context.Context, id string) error { + var ignored bool + return ec.c.CallContext(ctx, &ignored, "shh_deleteMessageFilter", id) +} + +// FilterMessages retrieves all messages that are received between the last call to +// this function and match the criteria that where given when the filter was created. +func (ec *Client) FilterMessages(ctx context.Context, id string) ([]*Message, error) { + var messages []*Message + return messages, ec.c.CallContext(ctx, &messages, "shh_getFilterMessages", id) +} diff --git a/whisper/whisperv5/config.go b/whisper/whisperv5/config.go new file mode 100644 index 0000000000..a53d504484 --- /dev/null +++ b/whisper/whisperv5/config.go @@ -0,0 +1,52 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package whisperv5 + +import ( + "gopkg.in/urfave/cli.v1" +) + +//go:generate gencodec -type Config -formats toml -out gen_config.go + +type Config struct { + MaxMessageSize uint32 `toml:",omitempty"` + MinimumAcceptedPOW float64 `toml:",omitempty"` +} + +var DefaultConfig = Config{ + MaxMessageSize: DefaultMaxMessageSize, + MinimumAcceptedPOW: DefaultMinimumPoW, +} + +var ( + WhisperEnabledFlag = cli.BoolFlag{ + Name: "shh", + Usage: "Enable Whisper", + } + MaxMessageSizeFlag = cli.IntFlag{ + Name: "shh.maxmessagesize", + Usage: "Max message size accepted", + Value: int(DefaultMaxMessageSize), + } + MinPOWFlag = cli.Float64Flag{ + Name: "shh.pow", + Usage: "Minimum POW accepted", + Value: DefaultMinimumPoW, + } + + Flags = []cli.Flag{WhisperEnabledFlag, MaxMessageSizeFlag, MinPOWFlag} +) diff --git a/whisper/whisperv5/doc.go b/whisper/whisperv5/doc.go index 768291a161..7a57488bd7 100644 --- a/whisper/whisperv5/doc.go +++ b/whisper/whisperv5/doc.go @@ -55,8 +55,9 @@ const ( AESNonceLength = 12 keyIdSize = 32 - DefaultMaxMessageLength = 1024 * 1024 - DefaultMinimumPoW = 0.2 + MaxMessageSize = uint32(10 * 1024 * 1024) // maximum accepted size of a message. + DefaultMaxMessageSize = uint32(1024 * 1024) + DefaultMinimumPoW = 0.2 padSizeLimit = 256 // just an arbitrary number, could be changed without breaking the protocol (must not exceed 2^24) messageQueueLimit = 1024 diff --git a/whisper/whisperv5/envelope.go b/whisper/whisperv5/envelope.go index d95fcab750..169cbba9dd 100644 --- a/whisper/whisperv5/envelope.go +++ b/whisper/whisperv5/envelope.go @@ -62,7 +62,7 @@ func (e *Envelope) rlpWithoutNonce() []byte { // NewEnvelope wraps a Whisper message with expiration and destination data // included into an envelope for network forwarding. -func NewEnvelope(ttl uint32, topic TopicType, aesNonce []byte, msg *SentMessage) *Envelope { +func NewEnvelope(ttl uint32, topic TopicType, aesNonce []byte, msg *sentMessage) *Envelope { env := Envelope{ Version: make([]byte, 1), Expiry: uint32(time.Now().Add(time.Second * time.Duration(ttl)).Unix()), diff --git a/whisper/whisperv5/filter.go b/whisper/whisperv5/filter.go index 03101d4a42..d571160d7f 100644 --- a/whisper/whisperv5/filter.go +++ b/whisper/whisperv5/filter.go @@ -163,6 +163,7 @@ func (f *Filter) Retrieve() (all []*ReceivedMessage) { for _, msg := range f.Messages { all = append(all, msg) } + f.Messages = make(map[common.Hash]*ReceivedMessage) // delete old messages return all } diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index dd4ab9e8db..4ce87eee22 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -97,7 +97,7 @@ func TestInstallFilters(t *testing.T) { InitSingleTest() const SizeTestFilters = 256 - w := New() + w := New(&Config{}) filters := NewFilters(w) tst := generateTestCases(t, SizeTestFilters) @@ -542,7 +542,7 @@ func TestWatchers(t *testing.T) { var x, firstID string var err error - w := New() + w := New(&Config{}) filters := NewFilters(w) tst := generateTestCases(t, NumFilters) for i = 0; i < NumFilters; i++ { diff --git a/whisper/whisperv5/gen_config.go b/whisper/whisperv5/gen_config.go new file mode 100644 index 0000000000..2f10dcdb84 --- /dev/null +++ b/whisper/whisperv5/gen_config.go @@ -0,0 +1,32 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package whisperv5 + +func (c Config) MarshalTOML() (interface{}, error) { + type Config struct { + MaxMessageSize uint32 `toml:",omitempty"` + MinimumAcceptedPOW float64 `toml:",omitempty"` + } + var enc Config + enc.MaxMessageSize = c.MaxMessageSize + enc.MinimumAcceptedPOW = c.MinimumAcceptedPOW + return &enc, nil +} + +func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { + type Config struct { + MaxMessageSize *uint32 `toml:",omitempty"` + MinimumAcceptedPOW *float64 `toml:",omitempty"` + } + var dec Config + if err := unmarshal(&dec); err != nil { + return err + } + if dec.MaxMessageSize != nil { + c.MaxMessageSize = *dec.MaxMessageSize + } + if dec.MinimumAcceptedPOW != nil { + c.MinimumAcceptedPOW = *dec.MinimumAcceptedPOW + } + return nil +} diff --git a/whisper/whisperv5/gen_criteria_json.go b/whisper/whisperv5/gen_criteria_json.go new file mode 100644 index 0000000000..8d3e9ee24b --- /dev/null +++ b/whisper/whisperv5/gen_criteria_json.go @@ -0,0 +1,62 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package whisperv5 + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +func (c Criteria) MarshalJSON() ([]byte, error) { + type Criteria struct { + SymKeyID string `json:"symKeyID"` + PrivateKeyID string `json:"privateKeyID"` + Sig hexutil.Bytes `json:"sig"` + MinPow float64 `json:"minPow"` + Topics []TopicType `json:"topics"` + AllowP2P bool `json:"allowP2P"` + } + var enc Criteria + enc.SymKeyID = c.SymKeyID + enc.PrivateKeyID = c.PrivateKeyID + enc.Sig = c.Sig + enc.MinPow = c.MinPow + enc.Topics = c.Topics + enc.AllowP2P = c.AllowP2P + return json.Marshal(&enc) +} + +func (c *Criteria) UnmarshalJSON(input []byte) error { + type Criteria struct { + SymKeyID *string `json:"symKeyID"` + PrivateKeyID *string `json:"privateKeyID"` + Sig hexutil.Bytes `json:"sig"` + MinPow *float64 `json:"minPow"` + Topics []TopicType `json:"topics"` + AllowP2P *bool `json:"allowP2P"` + } + var dec Criteria + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.SymKeyID != nil { + c.SymKeyID = *dec.SymKeyID + } + if dec.PrivateKeyID != nil { + c.PrivateKeyID = *dec.PrivateKeyID + } + if dec.Sig != nil { + c.Sig = dec.Sig + } + if dec.MinPow != nil { + c.MinPow = *dec.MinPow + } + if dec.Topics != nil { + c.Topics = dec.Topics + } + if dec.AllowP2P != nil { + c.AllowP2P = *dec.AllowP2P + } + return nil +} diff --git a/whisper/whisperv5/gen_message_json.go b/whisper/whisperv5/gen_message_json.go new file mode 100644 index 0000000000..26168225c9 --- /dev/null +++ b/whisper/whisperv5/gen_message_json.go @@ -0,0 +1,80 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package whisperv5 + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +func (m Message) MarshalJSON() ([]byte, error) { + type Message struct { + Sig hexutil.Bytes `json:"sig,omitempty"` + TTL uint32 `json:"ttl"` + Timestamp uint32 `json:"timestamp"` + Topic TopicType `json:"topic"` + Payload hexutil.Bytes `json:"payload"` + Padding hexutil.Bytes `json:"padding"` + PoW float64 `json:"pow"` + Hash hexutil.Bytes `json:"hash"` + Dst hexutil.Bytes `json:"recipientPublicKey,omitempty"` + } + var enc Message + enc.Sig = m.Sig + enc.TTL = m.TTL + enc.Timestamp = m.Timestamp + enc.Topic = m.Topic + enc.Payload = m.Payload + enc.Padding = m.Padding + enc.PoW = m.PoW + enc.Hash = m.Hash + enc.Dst = m.Dst + return json.Marshal(&enc) +} + +func (m *Message) UnmarshalJSON(input []byte) error { + type Message struct { + Sig hexutil.Bytes `json:"sig,omitempty"` + TTL *uint32 `json:"ttl"` + Timestamp *uint32 `json:"timestamp"` + Topic *TopicType `json:"topic"` + Payload hexutil.Bytes `json:"payload"` + Padding hexutil.Bytes `json:"padding"` + PoW *float64 `json:"pow"` + Hash hexutil.Bytes `json:"hash"` + Dst hexutil.Bytes `json:"recipientPublicKey,omitempty"` + } + var dec Message + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Sig != nil { + m.Sig = dec.Sig + } + if dec.TTL != nil { + m.TTL = *dec.TTL + } + if dec.Timestamp != nil { + m.Timestamp = *dec.Timestamp + } + if dec.Topic != nil { + m.Topic = *dec.Topic + } + if dec.Payload != nil { + m.Payload = dec.Payload + } + if dec.Padding != nil { + m.Padding = dec.Padding + } + if dec.PoW != nil { + m.PoW = *dec.PoW + } + if dec.Hash != nil { + m.Hash = dec.Hash + } + if dec.Dst != nil { + m.Dst = dec.Dst + } + return nil +} diff --git a/whisper/whisperv5/gen_newmessage_json.go b/whisper/whisperv5/gen_newmessage_json.go new file mode 100644 index 0000000000..0231cd9191 --- /dev/null +++ b/whisper/whisperv5/gen_newmessage_json.go @@ -0,0 +1,86 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package whisperv5 + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +func (n NewMessage) MarshalJSON() ([]byte, error) { + type NewMessage struct { + SymKeyID string `json:"symKeyID"` + PublicKey hexutil.Bytes `json:"pubKey"` + Sig string `json:"sig"` + TTL uint32 `json:"ttl"` + Topic TopicType `json:"topic"` + Payload hexutil.Bytes `json:"payload"` + Padding hexutil.Bytes `json:"padding"` + PowTime uint32 `json:"powTime"` + PowTarget float64 `json:"powTarget"` + TargetPeer string `json:"targetPeer"` + } + var enc NewMessage + enc.SymKeyID = n.SymKeyID + enc.PublicKey = n.PublicKey + enc.Sig = n.Sig + enc.TTL = n.TTL + enc.Topic = n.Topic + enc.Payload = n.Payload + enc.Padding = n.Padding + enc.PowTime = n.PowTime + enc.PowTarget = n.PowTarget + enc.TargetPeer = n.TargetPeer + return json.Marshal(&enc) +} + +func (n *NewMessage) UnmarshalJSON(input []byte) error { + type NewMessage struct { + SymKeyID *string `json:"symKeyID"` + PublicKey hexutil.Bytes `json:"pubKey"` + Sig *string `json:"sig"` + TTL *uint32 `json:"ttl"` + Topic *TopicType `json:"topic"` + Payload hexutil.Bytes `json:"payload"` + Padding hexutil.Bytes `json:"padding"` + PowTime *uint32 `json:"powTime"` + PowTarget *float64 `json:"powTarget"` + TargetPeer *string `json:"targetPeer"` + } + var dec NewMessage + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.SymKeyID != nil { + n.SymKeyID = *dec.SymKeyID + } + if dec.PublicKey != nil { + n.PublicKey = dec.PublicKey + } + if dec.Sig != nil { + n.Sig = *dec.Sig + } + if dec.TTL != nil { + n.TTL = *dec.TTL + } + if dec.Topic != nil { + n.Topic = *dec.Topic + } + if dec.Payload != nil { + n.Payload = dec.Payload + } + if dec.Padding != nil { + n.Padding = dec.Padding + } + if dec.PowTime != nil { + n.PowTime = *dec.PowTime + } + if dec.PowTarget != nil { + n.PowTarget = *dec.PowTarget + } + if dec.TargetPeer != nil { + n.TargetPeer = *dec.TargetPeer + } + return nil +} diff --git a/whisper/whisperv5/message.go b/whisper/whisperv5/message.go index 4ef469b515..c27535cd1a 100644 --- a/whisper/whisperv5/message.go +++ b/whisper/whisperv5/message.go @@ -49,7 +49,7 @@ type MessageParams struct { // SentMessage represents an end-user data packet to transmit through the // Whisper protocol. These are wrapped into Envelopes that need not be // understood by intermediate nodes, just forwarded. -type SentMessage struct { +type sentMessage struct { Raw []byte } @@ -87,8 +87,8 @@ func (msg *ReceivedMessage) isAsymmetricEncryption() bool { } // NewMessage creates and initializes a non-signed, non-encrypted Whisper message. -func NewSentMessage(params *MessageParams) (*SentMessage, error) { - msg := SentMessage{} +func NewSentMessage(params *MessageParams) (*sentMessage, error) { + msg := sentMessage{} msg.Raw = make([]byte, 1, len(params.Payload)+len(params.Padding)+signatureLength+padSizeLimit) msg.Raw[0] = 0 // set all the flags to zero err := msg.appendPadding(params) @@ -119,7 +119,7 @@ func intSize(i int) (s int) { // appendPadding appends the pseudorandom padding bytes and sets the padding flag. // The last byte contains the size of padding (thus, its size must not exceed 256). -func (msg *SentMessage) appendPadding(params *MessageParams) error { +func (msg *sentMessage) appendPadding(params *MessageParams) error { rawSize := len(params.Payload) + 1 if params.Src != nil { rawSize += signatureLength @@ -164,7 +164,7 @@ func (msg *SentMessage) appendPadding(params *MessageParams) error { // sign calculates and sets the cryptographic signature for the message, // also setting the sign flag. -func (msg *SentMessage) sign(key *ecdsa.PrivateKey) error { +func (msg *sentMessage) sign(key *ecdsa.PrivateKey) error { if isMessageSigned(msg.Raw[0]) { // this should not happen, but no reason to panic log.Error("failed to sign the message: already signed") @@ -183,7 +183,7 @@ func (msg *SentMessage) sign(key *ecdsa.PrivateKey) error { } // encryptAsymmetric encrypts a message with a public key. -func (msg *SentMessage) encryptAsymmetric(key *ecdsa.PublicKey) error { +func (msg *sentMessage) encryptAsymmetric(key *ecdsa.PublicKey) error { if !ValidatePublicKey(key) { return errors.New("invalid public key provided for asymmetric encryption") } @@ -196,7 +196,7 @@ func (msg *SentMessage) encryptAsymmetric(key *ecdsa.PublicKey) error { // encryptSymmetric encrypts a message with a topic key, using AES-GCM-256. // nonce size should be 12 bytes (see cipher.gcmStandardNonceSize). -func (msg *SentMessage) encryptSymmetric(key []byte) (nonce []byte, err error) { +func (msg *sentMessage) encryptSymmetric(key []byte) (nonce []byte, err error) { if !validateSymmetricKey(key) { return nil, errors.New("invalid key provided for symmetric encryption") } @@ -224,13 +224,12 @@ func (msg *SentMessage) encryptSymmetric(key []byte) (nonce []byte, err error) { } // Wrap bundles the message into an Envelope to transmit over the network. -func (msg *SentMessage) Wrap(options *MessageParams) (envelope *Envelope, err error) { +func (msg *sentMessage) Wrap(options *MessageParams) (envelope *Envelope, err error) { if options.TTL == 0 { options.TTL = DefaultTTL } if options.Src != nil { - err = msg.sign(options.Src) - if err != nil { + if err = msg.sign(options.Src); err != nil { return nil, err } } @@ -242,14 +241,12 @@ func (msg *SentMessage) Wrap(options *MessageParams) (envelope *Envelope, err er } else { err = errors.New("unable to encrypt the message: neither symmetric nor assymmetric key provided") } - if err != nil { return nil, err } envelope = NewEnvelope(options.TTL, options.Topic, nonce, msg) - err = envelope.Seal(options) - if err != nil { + if err = envelope.Seal(options); err != nil { return nil, err } return envelope, nil diff --git a/whisper/whisperv5/peer_test.go b/whisper/whisperv5/peer_test.go index d3cd63b0b2..bae2adb6f5 100644 --- a/whisper/whisperv5/peer_test.go +++ b/whisper/whisperv5/peer_test.go @@ -113,7 +113,7 @@ func initialize(t *testing.T) { for i := 0; i < NumNodes; i++ { var node TestNode - node.shh = New() + node.shh = New(&DefaultConfig) node.shh.SetMinimumPoW(0.00000001) node.shh.Start(nil) topics := make([]TopicType, 0) diff --git a/whisper/whisperv5/topic.go b/whisper/whisperv5/topic.go index 54d7422d17..5fea2f28bd 100644 --- a/whisper/whisperv5/topic.go +++ b/whisper/whisperv5/topic.go @@ -19,10 +19,11 @@ package whisperv5 import ( + "encoding/json" "fmt" - "strings" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" ) // Topic represents a cryptographically secure, probabilistic partial @@ -46,24 +47,19 @@ func (topic *TopicType) String() string { return string(common.ToHex(topic[:])) } +func (t *TopicType) MarshalJSON() ([]byte, error) { + return json.Marshal(hexutil.Bytes(t[:])) +} + // UnmarshalJSON parses a hex representation to a topic. func (t *TopicType) UnmarshalJSON(input []byte) error { - length := len(input) - if length >= 2 && input[0] == '"' && input[length-1] == '"' { - input = input[1 : length-1] + var data hexutil.Bytes + if err := json.Unmarshal(input, &data); err != nil { + return err } - // strip "0x" for length check - if len(input) > 1 && strings.ToLower(string(input[:2])) == "0x" { - input = input[2:] + if len(data) != TopicLength { + return fmt.Errorf("unmarshalJSON failed: topic must be exactly %d bytes(%d)", TopicLength, len(input)) } - // validate the length of the input - if len(input) != TopicLength*2 { - return fmt.Errorf("unmarshalJSON failed: topic must be exactly %d bytes", TopicLength) - } - b := common.FromHex(string(input)) - if b == nil { - return fmt.Errorf("unmarshalJSON failed: wrong topic format") - } - *t = BytesToTopic(b) + *t = BytesToTopic(data) return nil } diff --git a/whisper/whisperv5/topic_test.go b/whisper/whisperv5/topic_test.go index df566da362..63593963b1 100644 --- a/whisper/whisperv5/topic_test.go +++ b/whisper/whisperv5/topic_test.go @@ -66,37 +66,32 @@ var unmarshalTestsGood = []struct { topic TopicType data []byte }{ - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("0x00000000")}, - {topic: TopicType{0x00, 0x7f, 0x80, 0xff}, data: []byte("0x007f80ff")}, - {topic: TopicType{0xff, 0x80, 0x7f, 0x00}, data: []byte("0xff807f00")}, - {topic: TopicType{0xf2, 0x6e, 0x77, 0x79}, data: []byte("0xf26e7779")}, - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("00000000")}, - {topic: TopicType{0x00, 0x80, 0x01, 0x00}, data: []byte("00800100")}, - {topic: TopicType{0x00, 0x7f, 0x80, 0xff}, data: []byte("007f80ff")}, - {topic: TopicType{0xff, 0x80, 0x7f, 0x00}, data: []byte("ff807f00")}, - {topic: TopicType{0xf2, 0x6e, 0x77, 0x79}, data: []byte("f26e7779")}, + {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte(`"0x00000000"`)}, + {topic: TopicType{0x00, 0x7f, 0x80, 0xff}, data: []byte(`"0x007f80ff"`)}, + {topic: TopicType{0xff, 0x80, 0x7f, 0x00}, data: []byte(`"0xff807f00"`)}, + {topic: TopicType{0xf2, 0x6e, 0x77, 0x79}, data: []byte(`"0xf26e7779"`)}, } var unmarshalTestsBad = []struct { topic TopicType data []byte }{ - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("0x000000")}, - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("0x0000000")}, - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("0x000000000")}, - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("0x0000000000")}, - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("000000")}, - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("0000000")}, - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("000000000")}, - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("0000000000")}, - {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte("abcdefg0")}, + {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte(`"0x000000"`)}, + {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte(`"0x0000000"`)}, + {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte(`"0x000000000"`)}, + {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte(`"0x0000000000"`)}, + {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte(`"000000"`)}, + {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte(`"0000000"`)}, + {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte(`"000000000"`)}, + {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte(`"0000000000"`)}, + {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: []byte(`"abcdefg0"`)}, } var unmarshalTestsUgly = []struct { topic TopicType data []byte }{ - {topic: TopicType{0x01, 0x00, 0x00, 0x00}, data: []byte("00000001")}, + {topic: TopicType{0x01, 0x00, 0x00, 0x00}, data: []byte(`"0x00000001"`)}, } func TestUnmarshalTestsGood(t *testing.T) { @@ -104,7 +99,7 @@ func TestUnmarshalTestsGood(t *testing.T) { var top TopicType err := top.UnmarshalJSON(tst.data) if err != nil { - t.Fatalf("failed test %d. input: %v.", i, tst.data) + t.Fatalf("failed test %d. input: %v. err: %v", i, tst.data, err) } else if top != tst.topic { t.Fatalf("failed test %d: have %v, want %v.", i, t, tst.topic) } diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index f2aad08efb..cc6c65a9f8 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -26,6 +26,8 @@ import ( "sync" "time" + "sync/atomic" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -44,6 +46,38 @@ type Statistics struct { totalMessagesCleared int } +type settingType byte +type settingsMap map[settingType]interface{} + +const ( + minPowIdx settingType = iota // Minimal PoW required by the whisper node + maxMsgSizeIdx settingType = iota // Maximal message length allowed by the whisper node + OverflowIdx settingType = iota // Indicator of message queue overflow +) + +type settingsVault struct { + vaultMu sync.Mutex + vault atomic.Value +} + +func (s *settingsVault) get(idx settingType) interface{} { + m := s.vault.Load().(settingsMap) + return m[idx] +} + +func (s *settingsVault) store(idx settingType, val interface{}) { + s.vaultMu.Lock() + defer s.vaultMu.Unlock() + + m1 := s.vault.Load().(settingsMap) + m2 := make(settingsMap) + for k, v := range m1 { + m2[k] = v + } + m2[idx] = val + s.vault.Store(m2) +} + // Whisper represents a dark communication interface through the Ethereum // network, using its very own P2P communication layer. type Whisper struct { @@ -54,28 +88,27 @@ type Whisper struct { symKeys map[string][]byte // Symmetric key storage keyMu sync.RWMutex // Mutex associated with key storages + poolMu sync.RWMutex // Mutex to sync the message and expiration pools envelopes map[common.Hash]*Envelope // Pool of envelopes currently tracked by this node expirations map[uint32]*set.SetNonTS // Message expiration pool - poolMu sync.RWMutex // Mutex to sync the message and expiration pools - peers map[*Peer]struct{} // Set of currently active peers peerMu sync.RWMutex // Mutex to sync the active peer set + peers map[*Peer]struct{} // Set of currently active peers messageQueue chan *Envelope // Message queue for normal whisper messages p2pMsgQueue chan *Envelope // Message queue for peer-to-peer messages (not to be forwarded any further) quit chan struct{} // Channel used for graceful exit - minPoW float64 // Minimal PoW required by the whisper node - maxMsgLength int // Maximal message length allowed by the whisper node - overflow bool // Indicator of message queue overflow + settings settingsVault // holds configuration settings that can be dynamically changed - stats Statistics // Statistics of whisper node + statsMu sync.Mutex // guard stats + stats Statistics // Statistics of whisper node mailServer MailServer // MailServer interface } // New creates a Whisper client ready to communicate through the Ethereum P2P network. -func New() *Whisper { +func New(cfg *Config) *Whisper { whisper := &Whisper{ privateKeys: make(map[string]*ecdsa.PrivateKey), symKeys: make(map[string][]byte), @@ -85,22 +118,47 @@ func New() *Whisper { messageQueue: make(chan *Envelope, messageQueueLimit), p2pMsgQueue: make(chan *Envelope, messageQueueLimit), quit: make(chan struct{}), - minPoW: DefaultMinimumPoW, - maxMsgLength: DefaultMaxMessageLength, } + whisper.filters = NewFilters(whisper) + whisper.settings.vault.Store(make(settingsMap)) + whisper.settings.store(minPowIdx, cfg.MinimumAcceptedPOW) + whisper.settings.store(maxMsgSizeIdx, cfg.MaxMessageSize) + whisper.settings.store(OverflowIdx, false) + // p2p whisper sub protocol handler whisper.protocol = p2p.Protocol{ Name: ProtocolName, Version: uint(ProtocolVersion), Length: NumberOfMessageCodes, Run: whisper.HandlePeer, + NodeInfo: func() interface{} { + return map[string]interface{}{ + "version": ProtocolVersionStr, + "maxMessageSize": whisper.MaxMessageSize(), + "minimumPoW": whisper.MinPow(), + } + }, } return whisper } +func (w *Whisper) MinPow() float64 { + return w.settings.get(minPowIdx).(float64) +} + +// MaxMessageSize returns the maximum accepted message size. +func (w *Whisper) MaxMessageSize() uint32 { + return w.settings.get(maxMsgSizeIdx).(uint32) +} + +// Overflow returns an indication if the message queue is full. +func (w *Whisper) Overflow() bool { + return w.settings.get(OverflowIdx).(bool) +} + // APIs returns the RPC descriptors the Whisper implementation offers func (w *Whisper) APIs() []rpc.API { return []rpc.API{ @@ -129,12 +187,12 @@ func (w *Whisper) Version() uint { return w.protocol.Version } -// SetMaxMessageLength sets the maximal message length allowed by this node -func (w *Whisper) SetMaxMessageLength(val int) error { - if val <= 0 { - return fmt.Errorf("invalid message length: %d", val) +// SetMaxMessageSize sets the maximal message size allowed by this node +func (w *Whisper) SetMaxMessageSize(size uint32) error { + if size > MaxMessageSize { + return fmt.Errorf("message size too large [%d>%d]", size, MaxMessageSize) } - w.maxMsgLength = val + w.settings.store(maxMsgSizeIdx, uint32(size)) return nil } @@ -143,7 +201,7 @@ func (w *Whisper) SetMinimumPoW(val float64) error { if val <= 0.0 { return fmt.Errorf("invalid PoW: %f", val) } - w.minPoW = val + w.settings.store(minPowIdx, val) return nil } @@ -240,6 +298,20 @@ func (w *Whisper) DeleteKeyPair(key string) bool { return false } +// AddKeyPair imports a asymmetric private key and returns it identifier. +func (w *Whisper) AddKeyPair(key *ecdsa.PrivateKey) (string, error) { + id, err := GenerateRandomID() + if err != nil { + return "", fmt.Errorf("failed to generate ID: %s", err) + } + + w.keyMu.Lock() + w.privateKeys[id] = key + w.keyMu.Unlock() + + return id, nil +} + // HasKeyPair checks if the the whisper node is configured with the private key // of the specified public pair. func (w *Whisper) HasKeyPair(id string) bool { @@ -451,7 +523,7 @@ func (wh *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { log.Warn("message loop", "peer", p.peer.ID(), "err", err) return err } - if packet.Size > uint32(wh.maxMsgLength) { + if packet.Size > wh.MaxMessageSize() { log.Warn("oversized message received", "peer", p.peer.ID()) return errors.New("oversized message received") } @@ -532,7 +604,7 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) { } } - if envelope.size() > wh.maxMsgLength { + if uint32(envelope.size()) > wh.MaxMessageSize() { return false, fmt.Errorf("huge messages are not allowed [%x]", envelope.Hash()) } @@ -547,7 +619,7 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) { return false, fmt.Errorf("wrong size of AESNonce: %d bytes [env: %x]", aesNonceSize, envelope.Hash()) } - if envelope.PoW() < wh.minPoW { + if envelope.PoW() < wh.MinPow() { log.Debug("envelope with low PoW dropped", "PoW", envelope.PoW(), "hash", envelope.Hash().Hex()) return false, nil // drop envelope without error } @@ -571,7 +643,9 @@ func (wh *Whisper) add(envelope *Envelope) (bool, error) { log.Trace("whisper envelope already cached", "hash", envelope.Hash().Hex()) } else { log.Trace("cached whisper envelope", "hash", envelope.Hash().Hex()) + wh.statsMu.Lock() wh.stats.memoryUsed += envelope.size() + wh.statsMu.Unlock() wh.postEvent(envelope, false) // notify the local node about the new message if wh.mailServer != nil { wh.mailServer.Archive(envelope) @@ -600,13 +674,13 @@ func (w *Whisper) checkOverflow() { queueSize := len(w.messageQueue) if queueSize == messageQueueLimit { - if !w.overflow { - w.overflow = true + if !w.Overflow() { + w.settings.store(OverflowIdx, true) log.Warn("message queue overflow") } } else if queueSize <= messageQueueLimit/2 { - if w.overflow { - w.overflow = false + if w.Overflow() { + w.settings.store(OverflowIdx, false) log.Warn("message queue overflow fixed (back to normal)") } } @@ -653,6 +727,8 @@ func (w *Whisper) expire() { w.poolMu.Lock() defer w.poolMu.Unlock() + w.statsMu.Lock() + defer w.statsMu.Unlock() w.stats.reset() now := uint32(time.Now().Unix()) for expiry, hashSet := range w.expirations { @@ -673,17 +749,11 @@ func (w *Whisper) expire() { } // Stats returns the whisper node statistics. -func (w *Whisper) Stats() string { - result := fmt.Sprintf("Memory usage: %d bytes. Average messages cleared per expiry cycle: %d. Total messages cleared: %d.", - w.stats.memoryUsed, w.stats.totalMessagesCleared/w.stats.cycles, w.stats.totalMessagesCleared) - if w.stats.messagesCleared > 0 { - result += fmt.Sprintf(" Latest expiry cycle cleared %d messages (%d bytes).", - w.stats.messagesCleared, w.stats.memoryCleared) - } - if w.overflow { - result += " Message queue state: overflow." - } - return result +func (w *Whisper) Stats() Statistics { + w.statsMu.Lock() + defer w.statsMu.Unlock() + + return w.stats } // Envelopes retrieves all the messages currently pooled by the node. @@ -734,15 +804,6 @@ func (s *Statistics) reset() { s.messagesCleared = 0 } -// ValidateKeyID checks the format of key id. -func ValidateKeyID(id string) error { - const target = keyIdSize * 2 - if len(id) != target { - return fmt.Errorf("wrong size of key ID (expected %d bytes, got %d)", target, len(id)) - } - return nil -} - // ValidatePublicKey checks the format of the given public key. func ValidatePublicKey(k *ecdsa.PublicKey) bool { return k != nil && k.X != nil && k.Y != nil && k.X.Sign() != 0 && k.Y.Sign() != 0 diff --git a/whisper/whisperv5/whisper_test.go b/whisper/whisperv5/whisper_test.go index 225728c42e..4c03cb2345 100644 --- a/whisper/whisperv5/whisper_test.go +++ b/whisper/whisperv5/whisper_test.go @@ -21,10 +21,11 @@ import ( mrand "math/rand" "testing" "time" + "crypto/ecdsa" ) func TestWhisperBasic(t *testing.T) { - w := New() + w := New(&DefaultConfig) p := w.Protocols() shh := p[0] if shh.Name != ProtocolName { @@ -117,8 +118,39 @@ func TestWhisperBasic(t *testing.T) { } } +func TestWhisperAsymmetricKeyImport(t *testing.T) { + var ( + w = New(&DefaultConfig) + privateKeys []*ecdsa.PrivateKey + ) + + for i:=0; i < 50; i++ { + id, err := w.NewKeyPair() + if err != nil { + t.Fatalf("could not generate key: %v", err) + } + + pk, err := w.GetPrivateKey(id) + if err != nil { + t.Fatalf("could not export private key: %v", err) + } + + privateKeys = append(privateKeys, pk) + + if !w.DeleteKeyPair(id) { + t.Fatalf("could not delete private key") + } + } + + for _, pk := range privateKeys { + if _, err := w.AddKeyPair(pk); err != nil { + t.Fatalf("could not import private key: %v", err) + } + } +} + func TestWhisperIdentityManagement(t *testing.T) { - w := New() + w := New(&DefaultConfig) id1, err := w.NewKeyPair() if err != nil { t.Fatalf("failed to generate new key pair: %s.", err) @@ -240,7 +272,7 @@ func TestWhisperSymKeyManagement(t *testing.T) { var err error var k1, k2 []byte - w := New() + w := New(&DefaultConfig) id1 := string("arbitrary-string-1") id2 := string("arbitrary-string-2") @@ -443,7 +475,7 @@ func TestWhisperSymKeyManagement(t *testing.T) { func TestExpiry(t *testing.T) { InitSingleTest() - w := New() + w := New(&DefaultConfig) w.SetMinimumPoW(0.0000001) defer w.SetMinimumPoW(DefaultMinimumPoW) w.Start(nil) @@ -500,9 +532,9 @@ func TestExpiry(t *testing.T) { func TestCustomization(t *testing.T) { InitSingleTest() - w := New() + w := New(&DefaultConfig) defer w.SetMinimumPoW(DefaultMinimumPoW) - defer w.SetMaxMessageLength(DefaultMaxMessageLength) + defer w.SetMaxMessageSize(DefaultMaxMessageSize) w.Start(nil) defer w.Stop() @@ -547,13 +579,13 @@ func TestCustomization(t *testing.T) { if err != nil { t.Fatalf("failed Wrap with seed %d: %s.", seed, err) } - w.SetMaxMessageLength(env.size() - 1) + w.SetMaxMessageSize(uint32(env.size() - 1)) err = w.Send(env) if err == nil { t.Fatalf("successfully sent oversized envelope (seed %d): false positive.", seed) } - w.SetMaxMessageLength(DefaultMaxMessageLength) + w.SetMaxMessageSize(DefaultMaxMessageSize) err = w.Send(env) if err != nil { t.Fatalf("failed to send second envelope with seed %d: %s.", seed, err) From 36e7963467eb2eb3cceff9fe1960db17a388e458 Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Wed, 21 Jun 2017 10:08:47 +0200 Subject: [PATCH 02/43] whisper: move ShhClient to its own package --- whisper/{whisperv5 => shhclient}/client.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) rename whisper/{whisperv5 => shhclient}/client.go (93%) diff --git a/whisper/whisperv5/client.go b/whisper/shhclient/client.go similarity index 93% rename from whisper/whisperv5/client.go rename to whisper/shhclient/client.go index f53fe61433..61c1b7ab72 100644 --- a/whisper/whisperv5/client.go +++ b/whisper/shhclient/client.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package whisperv5 +package shhclient import ( "context" @@ -22,9 +22,10 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/rpc" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" ) -// Client defines typed wrappers for the Whisper RPC API. +// Client defines typed wrappers for the Whisper v5 RPC API. type Client struct { c *rpc.Client } @@ -51,8 +52,8 @@ func (sc *Client) Version(ctx context.Context) (uint, error) { } // Info returns diagnostic information about the whisper node. -func (sc *Client) Info(ctx context.Context) (Info, error) { - var info Info +func (sc *Client) Info(ctx context.Context) (whisper.Info, error) { + var info whisper.Info err := sc.c.CallContext(ctx, &info, "shh_info") return info, err } @@ -159,7 +160,7 @@ func (sc *Client) DeleteSymmetricKey(ctx context.Context, id string) error { } // Post a message onto the network. -func (sc *Client) Post(ctx context.Context, message NewMessage) error { +func (sc *Client) Post(ctx context.Context, message whisper.NewMessage) error { var ignored bool return sc.c.CallContext(ctx, &ignored, "shh_post", message) } @@ -167,14 +168,14 @@ func (sc *Client) Post(ctx context.Context, message NewMessage) error { // SubscribeMessages subscribes to messages that match the given criteria. This method // is only supported on bi-directional connections such as websockets and IPC. // NewMessageFilter uses polling and is supported over HTTP. -func (ec *Client) SubscribeMessages(ctx context.Context, criteria Criteria, ch chan<- *Message) (ethereum.Subscription, error) { +func (ec *Client) SubscribeMessages(ctx context.Context, criteria whisper.Criteria, ch chan<- *whisper.Message) (ethereum.Subscription, error) { return ec.c.ShhSubscribe(ctx, ch, "messages", criteria) } // NewMessageFilter creates a filter within the node. This filter can be used to poll // for new messages (see FilterMessages) that satisfy the given criteria. A filter can // timeout when it was polled for in whisper.filterTimeout. -func (ec *Client) NewMessageFilter(ctx context.Context, criteria Criteria) (string, error) { +func (ec *Client) NewMessageFilter(ctx context.Context, criteria whisper.Criteria) (string, error) { var id string return id, ec.c.CallContext(ctx, &id, "shh_newMessageFilter", criteria) } @@ -187,7 +188,7 @@ func (ec *Client) DeleteMessageFilter(ctx context.Context, id string) error { // FilterMessages retrieves all messages that are received between the last call to // this function and match the criteria that where given when the filter was created. -func (ec *Client) FilterMessages(ctx context.Context, id string) ([]*Message, error) { - var messages []*Message +func (ec *Client) FilterMessages(ctx context.Context, id string) ([]*whisper.Message, error) { + var messages []*whisper.Message return messages, ec.c.CallContext(ctx, &messages, "shh_getFilterMessages", id) } From 767dc6c73d652731198d96c68ea0e1d00bf8842d Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Wed, 21 Jun 2017 10:11:35 +0200 Subject: [PATCH 03/43] whisper: remove gencodec override for config --- whisper/whisperv5/config.go | 2 -- whisper/whisperv5/gen_config.go | 32 -------------------------------- 2 files changed, 34 deletions(-) delete mode 100644 whisper/whisperv5/gen_config.go diff --git a/whisper/whisperv5/config.go b/whisper/whisperv5/config.go index a53d504484..af59f1bdc3 100644 --- a/whisper/whisperv5/config.go +++ b/whisper/whisperv5/config.go @@ -20,8 +20,6 @@ import ( "gopkg.in/urfave/cli.v1" ) -//go:generate gencodec -type Config -formats toml -out gen_config.go - type Config struct { MaxMessageSize uint32 `toml:",omitempty"` MinimumAcceptedPOW float64 `toml:",omitempty"` diff --git a/whisper/whisperv5/gen_config.go b/whisper/whisperv5/gen_config.go deleted file mode 100644 index 2f10dcdb84..0000000000 --- a/whisper/whisperv5/gen_config.go +++ /dev/null @@ -1,32 +0,0 @@ -// Code generated by github.com/fjl/gencodec. DO NOT EDIT. - -package whisperv5 - -func (c Config) MarshalTOML() (interface{}, error) { - type Config struct { - MaxMessageSize uint32 `toml:",omitempty"` - MinimumAcceptedPOW float64 `toml:",omitempty"` - } - var enc Config - enc.MaxMessageSize = c.MaxMessageSize - enc.MinimumAcceptedPOW = c.MinimumAcceptedPOW - return &enc, nil -} - -func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { - type Config struct { - MaxMessageSize *uint32 `toml:",omitempty"` - MinimumAcceptedPOW *float64 `toml:",omitempty"` - } - var dec Config - if err := unmarshal(&dec); err != nil { - return err - } - if dec.MaxMessageSize != nil { - c.MaxMessageSize = *dec.MaxMessageSize - } - if dec.MinimumAcceptedPOW != nil { - c.MinimumAcceptedPOW = *dec.MinimumAcceptedPOW - } - return nil -} From 3d66ba56ef34e2b6a3a10c254506d8686adfff39 Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Wed, 21 Jun 2017 10:23:45 +0200 Subject: [PATCH 04/43] whisper: remove obsolete api tests --- whisper/whisperv5/api_test.go | 672 ---------------------------------- 1 file changed, 672 deletions(-) delete mode 100644 whisper/whisperv5/api_test.go diff --git a/whisper/whisperv5/api_test.go b/whisper/whisperv5/api_test.go deleted file mode 100644 index 3b2d9b2b68..0000000000 --- a/whisper/whisperv5/api_test.go +++ /dev/null @@ -1,672 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package whisperv5 - -/* -func TestBasic(t *testing.T) { - var id string = "test" - w := New() - api := NewPublicWhisperAPI(w) - if api == nil { - t.Fatalf("failed to create API.") - } - - ver, err := api.Version() - if err != nil { - t.Fatalf("failed generateFilter: %s.", err) - } - - if uint64(ver) != ProtocolVersion { - t.Fatalf("wrong version: %d.", ver) - } - - mail := api.GetNewSubscriptionMessages("non-existent-id") - if len(mail) != 0 { - t.Fatalf("failed GetFilterChanges: premature result") - } - - exist, err := api.HasKeyPair(id) - if err != nil { - t.Fatalf("failed initial HasIdentity: %s.", err) - } - if exist { - t.Fatalf("failed initial HasIdentity: false positive.") - } - - success, err := api.DeleteKeyPair(id) - if err != nil { - t.Fatalf("failed DeleteIdentity: %s.", err) - } - if success { - t.Fatalf("deleted non-existing identity: false positive.") - } - - pub, err := api.NewKeyPair() - if err != nil { - t.Fatalf("failed NewIdentity: %s.", err) - } - if len(pub) == 0 { - t.Fatalf("failed NewIdentity: empty") - } - - exist, err = api.HasKeyPair(pub) - if err != nil { - t.Fatalf("failed HasIdentity: %s.", err) - } - if !exist { - t.Fatalf("failed HasIdentity: false negative.") - } - - success, err = api.DeleteKeyPair(pub) - if err != nil { - t.Fatalf("failed to delete second identity: %s.", err) - } - if !success { - t.Fatalf("failed to delete second identity.") - } - - exist, err = api.HasKeyPair(pub) - if err != nil { - t.Fatalf("failed HasIdentity(): %s.", err) - } - if exist { - t.Fatalf("failed HasIdentity(): false positive.") - } - - id = "arbitrary text" - id2 := "another arbitrary string" - - exist, err = api.HasSymmetricKey(id) - if err != nil { - t.Fatalf("failed HasSymKey: %s.", err) - } - if exist { - t.Fatalf("failed HasSymKey: false positive.") - } - - id, err = api.GenerateSymmetricKey() - if err != nil { - t.Fatalf("failed GenerateSymKey: %s.", err) - } - - exist, err = api.HasSymmetricKey(id) - if err != nil { - t.Fatalf("failed HasSymKey(): %s.", err) - } - if !exist { - t.Fatalf("failed HasSymKey(): false negative.") - } - - const password = "some stuff here" - id, err = api.AddSymmetricKeyFromPassword(password) - if err != nil { - t.Fatalf("failed AddSymKey: %s.", err) - } - - id2, err = api.AddSymmetricKeyFromPassword(password) - if err != nil { - t.Fatalf("failed AddSymKey: %s.", err) - } - - exist, err = api.HasSymmetricKey(id2) - if err != nil { - t.Fatalf("failed HasSymKey(id2): %s.", err) - } - if !exist { - t.Fatalf("failed HasSymKey(id2): false negative.") - } - - k1, err := api.GetSymmetricKey(id) - if err != nil { - t.Fatalf("failed GetSymKey(id): %s.", err) - } - k2, err := api.GetSymmetricKey(id2) - if err != nil { - t.Fatalf("failed GetSymKey(id2): %s.", err) - } - - if !bytes.Equal(k1, k2) { - t.Fatalf("installed keys are not equal") - } - - exist, err = api.DeleteSymmetricKey(id) - if err != nil { - t.Fatalf("failed DeleteSymKey(id): %s.", err) - } - if !exist { - t.Fatalf("failed DeleteSymKey(id): false negative.") - } - - exist, err = api.HasSymmetricKey(id) - if err != nil { - t.Fatalf("failed HasSymKey(id): %s.", err) - } - if exist { - t.Fatalf("failed HasSymKey(id): false positive.") - } -} - -func TestUnmarshalFilterArgs(t *testing.T) { - s := []byte(`{ - "type":"sym", - "key":"0x70c87d191324e6712a591f304b4eedef6ad9bb9d", - "sig":"0x9b2055d370f73ec7d8a03e965129118dc8f5bf83", - "minPoW":2.34, - "topics":["0x00000000", "0x007f80ff", "0xff807f00", "0xf26e7779"], - "allowP2P":true - }`) - - var f WhisperFilterArgs - err := f.UnmarshalJSON(s) - if err != nil { - t.Fatalf("failed UnmarshalJSON: %s.", err) - } - - if !f.Symmetric { - t.Fatalf("wrong type.") - } - if f.Key != "0x70c87d191324e6712a591f304b4eedef6ad9bb9d" { - t.Fatalf("wrong key: %s.", f.Key) - } - if f.Sig != "0x9b2055d370f73ec7d8a03e965129118dc8f5bf83" { - t.Fatalf("wrong sig: %s.", f.Sig) - } - if f.MinPoW != 2.34 { - t.Fatalf("wrong MinPoW: %f.", f.MinPoW) - } - if !f.AllowP2P { - t.Fatalf("wrong AllowP2P.") - } - if len(f.Topics) != 4 { - t.Fatalf("wrong topics number: %d.", len(f.Topics)) - } - - i := 0 - if !bytes.Equal(f.Topics[i], []byte{0x00, 0x00, 0x00, 0x00}) { - t.Fatalf("wrong topic[%d]: %x.", i, f.Topics[i]) - } - - i++ - if !bytes.Equal(f.Topics[i], []byte{0x00, 0x7f, 0x80, 0xff}) { - t.Fatalf("wrong topic[%d]: %x.", i, f.Topics[i]) - } - - i++ - if !bytes.Equal(f.Topics[i], []byte{0xff, 0x80, 0x7f, 0x00}) { - t.Fatalf("wrong topic[%d]: %x.", i, f.Topics[i]) - } - - i++ - if !bytes.Equal(f.Topics[i], []byte{0xf2, 0x6e, 0x77, 0x79}) { - t.Fatalf("wrong topic[%d]: %x.", i, f.Topics[i]) - } -} - -func TestUnmarshalPostArgs(t *testing.T) { - s := []byte(`{ - "type":"sym", - "ttl":12345, - "sig":"0x70c87d191324e6712a591f304b4eedef6ad9bb9d", - "key":"0x9b2055d370f73ec7d8a03e965129118dc8f5bf83", - "topic":"0xf26e7779", - "padding":"0x74686973206973206D79207465737420737472696E67", - "payload":"0x7061796C6F61642073686F756C642062652070736575646F72616E646F6D", - "powTime":777, - "powTarget":3.1416, - "targetPeer":"enode://915533f667b1369793ebb9bda022416b1295235a1420799cd87a969467372546d808ebf59c5c9ce23f103d59b61b97df8af91f0908552485975397181b993461@127.0.0.1:12345" - }`) - - var a PostArgs - err := json.Unmarshal(s, &a) - if err != nil { - t.Fatalf("failed UnmarshalJSON: %s.", err) - } - - if a.Type != "sym" { - t.Fatalf("wrong Type: %s.", a.Type) - } - if a.TTL != 12345 { - t.Fatalf("wrong ttl: %d.", a.TTL) - } - if a.Sig != "0x70c87d191324e6712a591f304b4eedef6ad9bb9d" { - t.Fatalf("wrong From: %s.", a.Sig) - } - if a.Key != "0x9b2055d370f73ec7d8a03e965129118dc8f5bf83" { - t.Fatalf("wrong Key: %s.", a.Key) - } - - if BytesToTopic(a.Topic) != (TopicType{0xf2, 0x6e, 0x77, 0x79}) { - t.Fatalf("wrong topic: %x.", a.Topic) - } - if string(a.Padding) != "this is my test string" { - t.Fatalf("wrong Padding: %s.", string(a.Padding)) - } - if string(a.Payload) != "payload should be pseudorandom" { - t.Fatalf("wrong Payload: %s.", string(a.Payload)) - } - if a.PowTime != 777 { - t.Fatalf("wrong PowTime: %d.", a.PowTime) - } - if a.PowTarget != 3.1416 { - t.Fatalf("wrong PowTarget: %f.", a.PowTarget) - } - if a.TargetPeer != "enode://915533f667b1369793ebb9bda022416b1295235a1420799cd87a969467372546d808ebf59c5c9ce23f103d59b61b97df8af91f0908552485975397181b993461@127.0.0.1:12345" { - t.Fatalf("wrong PeerID: %s.", a.TargetPeer) - } -} - -func waitForMessages(api *PublicWhisperAPI, id string, target int) []*WhisperMessage { - // timeout: 2 seconds - result := make([]*WhisperMessage, 0, target) - for i := 0; i < 100; i++ { - mail := api.GetNewSubscriptionMessages(id) - if len(mail) > 0 { - for _, m := range mail { - result = append(result, m) - } - if len(result) >= target { - break - } - } - time.Sleep(time.Millisecond * 20) - } - - return result -} - -func TestIntegrationAsym(t *testing.T) { - w := New() - api := NewPublicWhisperAPI(w) - if api == nil { - t.Fatalf("failed to create API.") - } - - api.Start() - defer api.Stop() - - sig, err := api.NewKeyPair() - if err != nil { - t.Fatalf("failed NewIdentity: %s.", err) - } - if len(sig) == 0 { - t.Fatalf("wrong signature") - } - - exist, err := api.HasKeyPair(sig) - if err != nil { - t.Fatalf("failed HasIdentity: %s.", err) - } - if !exist { - t.Fatalf("failed HasIdentity: false negative.") - } - - sigPubKey, err := api.GetPublicKey(sig) - if err != nil { - t.Fatalf("failed GetPublicKey: %s.", err) - } - - key, err := api.NewKeyPair() - if err != nil { - t.Fatalf("failed NewIdentity(): %s.", err) - } - if len(key) == 0 { - t.Fatalf("wrong key") - } - - dstPubKey, err := api.GetPublicKey(key) - if err != nil { - t.Fatalf("failed GetPublicKey: %s.", err) - } - - var topics [2]TopicType - topics[0] = TopicType{0x00, 0x64, 0x00, 0xff} - topics[1] = TopicType{0xf2, 0x6e, 0x77, 0x79} - var f WhisperFilterArgs - f.Symmetric = false - f.Key = key - f.Sig = sigPubKey.String() - f.Topics = make([][]byte, 2) - f.Topics[0] = topics[0][:] - f.Topics[1] = topics[1][:] - f.MinPoW = DefaultMinimumPoW / 2 - f.AllowP2P = true - - id, err := api.Subscribe(f) - if err != nil { - t.Fatalf("failed to create new filter: %s.", err) - } - - var p PostArgs - p.Type = "asym" - p.TTL = 2 - p.Sig = sig - p.Key = dstPubKey.String() - p.Padding = []byte("test string") - p.Payload = []byte("extended test string") - p.PowTarget = DefaultMinimumPoW - p.PowTime = 2 - p.Topic = hexutil.Bytes{0xf2, 0x6e, 0x77, 0x79} // topics[1] - - err = api.Post(p) - if err != nil { - t.Errorf("failed to post message: %s.", err) - } - - mail := waitForMessages(api, id, 1) - if len(mail) != 1 { - t.Fatalf("failed to GetFilterChanges: got %d messages.", len(mail)) - } - - text := string(common.FromHex(mail[0].Payload)) - if text != string("extended test string") { - t.Fatalf("failed to decrypt first message: %s.", text) - } - - p.Padding = []byte("new value") - p.Payload = []byte("extended new value") - err = api.Post(p) - if err != nil { - t.Fatalf("failed to post next message: %s.", err) - } - - mail = waitForMessages(api, id, 1) - if len(mail) != 1 { - t.Fatalf("failed to GetFilterChanges: got %d messages.", len(mail)) - } - - text = string(common.FromHex(mail[0].Payload)) - if text != string("extended new value") { - t.Fatalf("failed to decrypt second message: %s.", text) - } -} - -func TestIntegrationSym(t *testing.T) { - w := New() - api := NewPublicWhisperAPI(w) - if api == nil { - t.Fatalf("failed to create API.") - } - - api.Start() - defer api.Stop() - - symKeyID, err := api.GenerateSymmetricKey() - if err != nil { - t.Fatalf("failed GenerateSymKey: %s.", err) - } - - sig, err := api.NewKeyPair() - if err != nil { - t.Fatalf("failed NewKeyPair: %s.", err) - } - if len(sig) == 0 { - t.Fatalf("wrong signature") - } - - sigPubKey, err := api.GetPublicKey(sig) - if err != nil { - t.Fatalf("failed GetPublicKey: %s.", err) - } - - exist, err := api.HasKeyPair(sig) - if err != nil { - t.Fatalf("failed HasIdentity: %s.", err) - } - if !exist { - t.Fatalf("failed HasIdentity: false negative.") - } - - var topics [2]TopicType - topics[0] = TopicType{0x00, 0x7f, 0x80, 0xff} - topics[1] = TopicType{0xf2, 0x6e, 0x77, 0x79} - var f WhisperFilterArgs - f.Symmetric = true - f.Key = symKeyID - f.Topics = make([][]byte, 2) - f.Topics[0] = topics[0][:] - f.Topics[1] = topics[1][:] - f.MinPoW = DefaultMinimumPoW / 2 - f.Sig = sigPubKey.String() - f.AllowP2P = false - - id, err := api.Subscribe(f) - if err != nil { - t.Fatalf("failed to create new filter: %s.", err) - } - - var p PostArgs - p.Type = "sym" - p.TTL = 1 - p.Key = symKeyID - p.Sig = sig - p.Padding = []byte("test string") - p.Payload = []byte("extended test string") - p.PowTarget = DefaultMinimumPoW - p.PowTime = 2 - p.Topic = hexutil.Bytes{0xf2, 0x6e, 0x77, 0x79} - - err = api.Post(p) - if err != nil { - t.Fatalf("failed to post first message: %s.", err) - } - - mail := waitForMessages(api, id, 1) - if len(mail) != 1 { - t.Fatalf("failed GetFilterChanges: got %d messages.", len(mail)) - } - - text := string(common.FromHex(mail[0].Payload)) - if text != string("extended test string") { - t.Fatalf("failed to decrypt first message: %s.", text) - } - - p.Padding = []byte("new value") - p.Payload = []byte("extended new value") - err = api.Post(p) - if err != nil { - t.Fatalf("failed to post second message: %s.", err) - } - - mail = waitForMessages(api, id, 1) - if len(mail) != 1 { - t.Fatalf("failed second GetFilterChanges: got %d messages.", len(mail)) - } - - text = string(common.FromHex(mail[0].Payload)) - if text != string("extended new value") { - t.Fatalf("failed to decrypt second message: %s.", text) - } -} - -func TestIntegrationSymWithFilter(t *testing.T) { - w := New() - api := NewPublicWhisperAPI(w) - if api == nil { - t.Fatalf("failed to create API.") - } - - api.Start() - defer api.Stop() - - symKeyID, err := api.GenerateSymmetricKey() - if err != nil { - t.Fatalf("failed to GenerateSymKey: %s.", err) - } - - sigKeyID, err := api.NewKeyPair() - if err != nil { - t.Fatalf("failed NewIdentity: %s.", err) - } - if len(sigKeyID) == 0 { - t.Fatalf("wrong signature.") - } - - exist, err := api.HasKeyPair(sigKeyID) - if err != nil { - t.Fatalf("failed HasIdentity: %s.", err) - } - if !exist { - t.Fatalf("failed HasIdentity: does not exist.") - } - - sigPubKey, err := api.GetPublicKey(sigKeyID) - if err != nil { - t.Fatalf("failed GetPublicKey: %s.", err) - } - - var topics [2]TopicType - topics[0] = TopicType{0x00, 0x7f, 0x80, 0xff} - topics[1] = TopicType{0xf2, 0x6e, 0x77, 0x79} - var f WhisperFilterArgs - f.Symmetric = true - f.Key = symKeyID - f.Topics = make([][]byte, 2) - f.Topics[0] = topics[0][:] - f.Topics[1] = topics[1][:] - f.MinPoW = DefaultMinimumPoW / 2 - f.Sig = sigPubKey.String() - f.AllowP2P = false - - id, err := api.Subscribe(f) - if err != nil { - t.Fatalf("failed to create new filter: %s.", err) - } - - var p PostArgs - p.Type = "sym" - p.TTL = 1 - p.Key = symKeyID - p.Sig = sigKeyID - p.Padding = []byte("test string") - p.Payload = []byte("extended test string") - p.PowTarget = DefaultMinimumPoW - p.PowTime = 2 - p.Topic = hexutil.Bytes{0xf2, 0x6e, 0x77, 0x79} - - err = api.Post(p) - if err != nil { - t.Fatalf("failed to post message: %s.", err) - } - - mail := waitForMessages(api, id, 1) - if len(mail) != 1 { - t.Fatalf("failed to GetFilterChanges: got %d messages.", len(mail)) - } - - text := string(common.FromHex(mail[0].Payload)) - if text != string("extended test string") { - t.Fatalf("failed to decrypt first message: %s.", text) - } - - p.Padding = []byte("new value") - p.Payload = []byte("extended new value") - err = api.Post(p) - if err != nil { - t.Fatalf("failed to post next message: %s.", err) - } - - mail = waitForMessages(api, id, 1) - if len(mail) != 1 { - t.Fatalf("failed to GetFilterChanges: got %d messages.", len(mail)) - } - - text = string(common.FromHex(mail[0].Payload)) - if text != string("extended new value") { - t.Fatalf("failed to decrypt second message: %s.", text) - } -} - -func TestKey(t *testing.T) { - w := New() - api := NewPublicWhisperAPI(w) - if api == nil { - t.Fatalf("failed to create API.") - } - - k, err := api.AddSymmetricKeyFromPassword("wwww") - if err != nil { - t.Fatalf("failed to create key: %s.", err) - } - - s, err := api.GetSymmetricKey(k) - if err != nil { - t.Fatalf("failed to get sym key: %s.", err) - } - - k2, err := api.AddSymmetricKeyDirect(s) - if err != nil { - t.Fatalf("failed to add sym key: %s.", err) - } - - s2, err := api.GetSymmetricKey(k2) - if err != nil { - t.Fatalf("failed to get sym key: %s.", err) - } - - if s.String() != "0x448652d595bd6ec00b2a9ea220ad6c26592d9bf4cf79023d3c1b30cb681e6e07" { - t.Fatalf("wrong key from password: %s", s.String()) - } - - if !bytes.Equal(s, s2) { - t.Fatalf("wrong key") - } -} - -func TestSubscribe(t *testing.T) { - var err error - var s string - - w := New() - api := NewPublicWhisperAPI(w) - if api == nil { - t.Fatalf("failed to create API.") - } - - symKeyID, err := api.GenerateSymmetricKey() - if err != nil { - t.Fatalf("failed to GenerateSymKey: %s.", err) - } - - var f WhisperFilterArgs - f.Symmetric = true - f.Key = symKeyID - f.Topics = make([][]byte, 5) - f.Topics[0] = []byte{0x21} - f.Topics[1] = []byte{0xd2, 0xe3} - f.Topics[2] = []byte{0x64, 0x75, 0x76} - f.Topics[3] = []byte{0xf8, 0xe9, 0xa0, 0xba} - f.Topics[4] = []byte{0xcb, 0x3c, 0xdd, 0xee, 0xff} - - s, err = api.Subscribe(f) - if err == nil { - t.Fatalf("Subscribe: false positive.") - } - - f.Topics[4] = []byte{} - if err == nil { - t.Fatalf("Subscribe: false positive again.") - } - - f.Topics[4] = []byte{0x00} - s, err = api.Subscribe(f) - if err != nil { - t.Fatalf("failed to subscribe: %s.", err) - } else { - api.Unsubscribe(s) - } -} -*/ From b6b0e0019810dd5fad992891c5db051b8603292d Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Wed, 21 Jun 2017 10:24:34 +0200 Subject: [PATCH 05/43] whisper: fallback to default config if none is specified --- whisper/whisperv5/whisper.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index cc6c65a9f8..83a408dfcd 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -109,6 +109,10 @@ type Whisper struct { // New creates a Whisper client ready to communicate through the Ethereum P2P network. func New(cfg *Config) *Whisper { + if cfg == nil { + cfg = &DefaultConfig + } + whisper := &Whisper{ privateKeys: make(map[string]*ecdsa.PrivateKey), symKeys: make(map[string][]byte), From 4a1d516d78f61937d850c6622bc26955b5103a23 Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Wed, 21 Jun 2017 10:32:36 +0200 Subject: [PATCH 06/43] whisper: renamed errors --- whisper/whisperv5/api.go | 42 +++++++++++++++---------------- whisper/whisperv5/whisper.go | 2 +- whisper/whisperv5/whisper_test.go | 6 ++--- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/whisper/whisperv5/api.go b/whisper/whisperv5/api.go index fd25a4206c..f3d25f781d 100644 --- a/whisper/whisperv5/api.go +++ b/whisper/whisperv5/api.go @@ -18,11 +18,11 @@ package whisperv5 import ( "context" + "crypto/ecdsa" "errors" "fmt" - "time" - "crypto/ecdsa" "sync" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -37,12 +37,12 @@ const ( ) var ( - SymAsymErr = errors.New("specify either a symetric or a asymmetric key") - InvalidSymmetricKeyErr = errors.New("invalid symmetric key") - InvalidPublicKeyErr = errors.New("invalid public key") - InvalidSigningPubKey = errors.New("invalid signing public key") - TooLowPoWErr = errors.New("message rejected, PoW too low") - NoTopicsErr = errors.New("missing topic(s)") + ErrSymAsym = errors.New("specify either a symetric or a asymmetric key") + ErrInvalidSymmetricKey = errors.New("invalid symmetric key") + ErrInvalidPublicKey = errors.New("invalid public key") + ErrInvalidSigningPubKey = errors.New("invalid signing public key") + ErrTooLowPoW = errors.New("message rejected, PoW too low") + ErrNoTopics = errors.New("missing topic(s)") ) // PublicWhisperAPI provides the whisper RPC service that can be @@ -245,7 +245,7 @@ func (api *PublicWhisperAPI) Post(ctx context.Context, req NewMessage) (bool, er // user must specify either a symmetric or a asymmetric key if (symKeyGiven && pubKeyGiven) || (!symKeyGiven && !pubKeyGiven) { - return false, SymAsymErr + return false, ErrSymAsym } params := &MessageParams{ @@ -267,13 +267,13 @@ func (api *PublicWhisperAPI) Post(ctx context.Context, req NewMessage) (bool, er // Set symmetric key that is used to encrypt the message if symKeyGiven { if params.Topic == (TopicType{}) { // topics are mandatory with symmetric encryption - return false, NoTopicsErr + return false, ErrNoTopics } if params.KeySym, err = api.w.GetSymKey(req.SymKeyID); err != nil { return false, err } if !validateSymmetricKey(params.KeySym) { - return false, InvalidSymmetricKeyErr + return false, ErrInvalidSymmetricKey } } @@ -281,7 +281,7 @@ func (api *PublicWhisperAPI) Post(ctx context.Context, req NewMessage) (bool, er if pubKeyGiven { params.Dst = crypto.ToECDSAPub(req.PublicKey) if !ValidatePublicKey(params.Dst) { - return false, InvalidPublicKeyErr + return false, ErrInvalidPublicKey } } @@ -307,7 +307,7 @@ func (api *PublicWhisperAPI) Post(ctx context.Context, req NewMessage) (bool, er // ensure that the message PoW meets the node's minimum accepted PoW if req.PowTarget < api.w.MinPow() { - return false, TooLowPoWErr + return false, ErrTooLowPoW } return true, api.w.Send(env) @@ -346,7 +346,7 @@ func (api *PublicWhisperAPI) Messages(ctx context.Context, crit Criteria) (*rpc. // user must specify either a symmetric or a asymmetric key if (symKeyGiven && pubKeyGiven) || (!symKeyGiven && !pubKeyGiven) { - return nil, SymAsymErr + return nil, ErrSymAsym } filter := Filter{ @@ -358,7 +358,7 @@ func (api *PublicWhisperAPI) Messages(ctx context.Context, crit Criteria) (*rpc. if len(crit.Sig) > 0 { filter.Src = crypto.ToECDSAPub(crit.Sig) if !ValidatePublicKey(filter.Src) { - return nil, InvalidSigningPubKey + return nil, ErrInvalidSigningPubKey } } @@ -372,14 +372,14 @@ func (api *PublicWhisperAPI) Messages(ctx context.Context, crit Criteria) (*rpc. // listen for message that are encrypted with the given symmetric key if symKeyGiven { if len(filter.Topics) == 0 { - return nil, NoTopicsErr + return nil, ErrNoTopics } key, err := api.w.GetSymKey(crit.SymKeyID) if err != nil { return nil, err } if !validateSymmetricKey(key) { - return nil, InvalidSymmetricKeyErr + return nil, ErrInvalidSymmetricKey } filter.KeySym = key filter.SymKeyHash = crypto.Keccak256Hash(filter.KeySym) @@ -389,7 +389,7 @@ func (api *PublicWhisperAPI) Messages(ctx context.Context, crit Criteria) (*rpc. if pubKeyGiven { filter.KeyAsym, err = api.w.GetPrivateKey(crit.PrivateKeyID) if err != nil || filter.KeyAsym == nil { - return nil, InvalidPublicKeyErr + return nil, ErrInvalidPublicKey } } @@ -536,13 +536,13 @@ func (api *PublicWhisperAPI) NewMessageFilter(req Criteria) (string, error) { // user must specify either a symmetric or a asymmetric key if (symKeyGiven && asymKeyGiven) || (!symKeyGiven && !asymKeyGiven) { - return "", SymAsymErr + return "", ErrSymAsym } if len(req.Sig) > 0 { src = crypto.ToECDSAPub(req.Sig) if !ValidatePublicKey(src) { - return "", InvalidSigningPubKey + return "", ErrInvalidSigningPubKey } } @@ -551,7 +551,7 @@ func (api *PublicWhisperAPI) NewMessageFilter(req Criteria) (string, error) { return "", err } if !validateSymmetricKey(keySym) { - return "", InvalidSymmetricKeyErr + return "", ErrInvalidSymmetricKey } } diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index 83a408dfcd..1e10b1d76f 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -112,7 +112,7 @@ func New(cfg *Config) *Whisper { if cfg == nil { cfg = &DefaultConfig } - + whisper := &Whisper{ privateKeys: make(map[string]*ecdsa.PrivateKey), symKeys: make(map[string][]byte), diff --git a/whisper/whisperv5/whisper_test.go b/whisper/whisperv5/whisper_test.go index 4c03cb2345..1451438336 100644 --- a/whisper/whisperv5/whisper_test.go +++ b/whisper/whisperv5/whisper_test.go @@ -18,10 +18,10 @@ package whisperv5 import ( "bytes" + "crypto/ecdsa" mrand "math/rand" "testing" "time" - "crypto/ecdsa" ) func TestWhisperBasic(t *testing.T) { @@ -120,11 +120,11 @@ func TestWhisperBasic(t *testing.T) { func TestWhisperAsymmetricKeyImport(t *testing.T) { var ( - w = New(&DefaultConfig) + w = New(&DefaultConfig) privateKeys []*ecdsa.PrivateKey ) - for i:=0; i < 50; i++ { + for i := 0; i < 50; i++ { id, err := w.NewKeyPair() if err != nil { t.Fatalf("could not generate key: %v", err) From 7a11e86442ab51401ad764777666133d82656264 Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Wed, 21 Jun 2017 10:49:14 +0200 Subject: [PATCH 07/43] whisper: move flags from whisper package to utils --- cmd/geth/config.go | 14 +++++++------- cmd/geth/main.go | 9 +++++++-- cmd/geth/usage.go | 3 +-- cmd/utils/flags.go | 22 ++++++++++++++++++---- whisper/whisperv5/config.go | 23 +---------------------- 5 files changed, 34 insertions(+), 37 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 6a0183204e..d3600f1416 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -43,7 +43,7 @@ var ( Name: "dumpconfig", Usage: "Show configuration values", ArgsUsage: "", - Flags: append(append(nodeFlags, rpcFlags...), whisper.Flags...), + Flags: append(append(nodeFlags, rpcFlags...), whisperFlags...), Category: "MISCELLANEOUS COMMANDS", Description: `The dumpconfig command shows configuration values.`, } @@ -140,7 +140,7 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // enableWhisper returns true in case one of the whisper flags is set. func enableWhisper(ctx *cli.Context) bool { - for _, flag := range whisper.Flags { + for _, flag := range whisperFlags { if ctx.GlobalIsSet(flag.GetName()) { return true } @@ -155,13 +155,13 @@ func makeFullNode(ctx *cli.Context) *node.Node { // Whisper must be explicitly enabled by specifying at least 1 whisper flag or in dev mode shhEnabled := enableWhisper(ctx) - shhAutoEnabled := !ctx.GlobalIsSet(whisper.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name) + shhAutoEnabled := !ctx.GlobalIsSet(utils.WhisperEnabledFlag.Name) && ctx.GlobalIsSet(utils.DevModeFlag.Name) if shhEnabled || shhAutoEnabled { - if ctx.GlobalIsSet(whisper.MaxMessageSizeFlag.Name) { - cfg.Shh.MaxMessageSize = uint32(ctx.Int(whisper.MaxMessageSizeFlag.Name)) + if ctx.GlobalIsSet(utils.WhisperMaxMessageSizeFlag.Name) { + cfg.Shh.MaxMessageSize = uint32(ctx.Int(utils.WhisperMaxMessageSizeFlag.Name)) } - if ctx.GlobalIsSet(whisper.MinPOWFlag.Name) { - cfg.Shh.MinimumAcceptedPOW = ctx.Float64(whisper.MinPOWFlag.Name) + if ctx.GlobalIsSet(utils.WhisperMinPOWFlag.Name) { + cfg.Shh.MinimumAcceptedPOW = ctx.Float64(utils.WhisperMinPOWFlag.Name) } utils.RegisterShhService(stack, &cfg.Shh) } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index c5f783a082..ea87a32243 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -35,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" "gopkg.in/urfave/cli.v1" ) @@ -125,6 +124,12 @@ var ( utils.IPCDisabledFlag, utils.IPCPathFlag, } + + whisperFlags = []cli.Flag{ + utils.WhisperEnabledFlag, + utils.WhisperMaxMessageSizeFlag, + utils.WhisperMinPOWFlag, + } ) func init() { @@ -161,7 +166,7 @@ func init() { app.Flags = append(app.Flags, rpcFlags...) app.Flags = append(app.Flags, consoleFlags...) app.Flags = append(app.Flags, debug.Flags...) - app.Flags = append(app.Flags, whisper.Flags...) + app.Flags = append(app.Flags, whisperFlags...) app.Before = func(ctx *cli.Context) error { runtime.GOMAXPROCS(runtime.NumCPU()) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 4a3e22af8b..bb14de747b 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/internal/debug" - "github.com/ethereum/go-ethereum/whisper/whisperv5" "gopkg.in/urfave/cli.v1" ) @@ -190,7 +189,7 @@ var AppHelpFlagGroups = []flagGroup{ }, { Name: "Whisper (EXPERIMENTAL)", - Flags: whisperv5.Flags, + Flags: whisperFlags, }, { Name: "DEPRECATED", diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 095a9f62c6..b5a593ab6e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -458,6 +458,20 @@ var ( Usage: "Suggested gas price is the given percentile of a set of recent transaction gas prices", Value: eth.DefaultConfig.GPO.Percentile, } + WhisperEnabledFlag = cli.BoolFlag{ + Name: "shh", + Usage: "Enable Whisper", + } + WhisperMaxMessageSizeFlag = cli.IntFlag{ + Name: "shh.maxmessagesize", + Usage: "Max message size accepted", + Value: int(whisper.DefaultMaxMessageSize), + } + WhisperMinPOWFlag = cli.Float64Flag{ + Name: "shh.pow", + Usage: "Minimum POW accepted", + Value: whisper.DefaultMinimumPoW, + } ) // MakeDataDir retrieves the currently requested data directory, terminating @@ -875,11 +889,11 @@ func checkExclusive(ctx *cli.Context, flags ...cli.Flag) { // SetShhConfig applies shh-related command line flags to the config. func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { - if ctx.GlobalIsSet(whisper.MaxMessageSizeFlag.Name) { - cfg.MaxMessageSize = uint32(ctx.GlobalUint(whisper.MaxMessageSizeFlag.Name)) + if ctx.GlobalIsSet(WhisperMaxMessageSizeFlag.Name) { + cfg.MaxMessageSize = uint32(ctx.GlobalUint(WhisperMaxMessageSizeFlag.Name)) } - if ctx.GlobalIsSet(whisper.MinPOWFlag.Name) { - cfg.MinimumAcceptedPOW = ctx.GlobalFloat64(whisper.MinPOWFlag.Name) + if ctx.GlobalIsSet(WhisperMinPOWFlag.Name) { + cfg.MinimumAcceptedPOW = ctx.GlobalFloat64(WhisperMinPOWFlag.Name) } } diff --git a/whisper/whisperv5/config.go b/whisper/whisperv5/config.go index af59f1bdc3..290bf8962b 100644 --- a/whisper/whisperv5/config.go +++ b/whisper/whisperv5/config.go @@ -16,10 +16,6 @@ package whisperv5 -import ( - "gopkg.in/urfave/cli.v1" -) - type Config struct { MaxMessageSize uint32 `toml:",omitempty"` MinimumAcceptedPOW float64 `toml:",omitempty"` @@ -30,21 +26,4 @@ var DefaultConfig = Config{ MinimumAcceptedPOW: DefaultMinimumPoW, } -var ( - WhisperEnabledFlag = cli.BoolFlag{ - Name: "shh", - Usage: "Enable Whisper", - } - MaxMessageSizeFlag = cli.IntFlag{ - Name: "shh.maxmessagesize", - Usage: "Max message size accepted", - Value: int(DefaultMaxMessageSize), - } - MinPOWFlag = cli.Float64Flag{ - Name: "shh.pow", - Usage: "Minimum POW accepted", - Value: DefaultMinimumPoW, - } - - Flags = []cli.Flag{WhisperEnabledFlag, MaxMessageSizeFlag, MinPOWFlag} -) +var () From a4e4c76cb30f34f8a06431802fdc3b9ba0643d47 Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Wed, 21 Jun 2017 11:15:47 +0200 Subject: [PATCH 08/43] whisper: use syncmap for dynamic configuration options --- vendor/golang.org/x/sync/LICENSE | 27 ++ vendor/golang.org/x/sync/PATENTS | 22 ++ vendor/golang.org/x/sync/syncmap/map.go | 372 ++++++++++++++++++++++++ vendor/vendor.json | 6 + whisper/whisperv5/whisper.go | 61 ++-- 5 files changed, 445 insertions(+), 43 deletions(-) create mode 100644 vendor/golang.org/x/sync/LICENSE create mode 100644 vendor/golang.org/x/sync/PATENTS create mode 100644 vendor/golang.org/x/sync/syncmap/map.go diff --git a/vendor/golang.org/x/sync/LICENSE b/vendor/golang.org/x/sync/LICENSE new file mode 100644 index 0000000000..6a66aea5ea --- /dev/null +++ b/vendor/golang.org/x/sync/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/sync/PATENTS b/vendor/golang.org/x/sync/PATENTS new file mode 100644 index 0000000000..733099041f --- /dev/null +++ b/vendor/golang.org/x/sync/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/sync/syncmap/map.go b/vendor/golang.org/x/sync/syncmap/map.go new file mode 100644 index 0000000000..80e15847ef --- /dev/null +++ b/vendor/golang.org/x/sync/syncmap/map.go @@ -0,0 +1,372 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package syncmap provides a concurrent map implementation. +// It is a prototype for a proposed addition to the sync package +// in the standard library. +// (https://golang.org/issue/18177) +package syncmap + +import ( + "sync" + "sync/atomic" + "unsafe" +) + +// Map is a concurrent map with amortized-constant-time loads, stores, and deletes. +// It is safe for multiple goroutines to call a Map's methods concurrently. +// +// The zero Map is valid and empty. +// +// A Map must not be copied after first use. +type Map struct { + mu sync.Mutex + + // read contains the portion of the map's contents that are safe for + // concurrent access (with or without mu held). + // + // The read field itself is always safe to load, but must only be stored with + // mu held. + // + // Entries stored in read may be updated concurrently without mu, but updating + // a previously-expunged entry requires that the entry be copied to the dirty + // map and unexpunged with mu held. + read atomic.Value // readOnly + + // dirty contains the portion of the map's contents that require mu to be + // held. To ensure that the dirty map can be promoted to the read map quickly, + // it also includes all of the non-expunged entries in the read map. + // + // Expunged entries are not stored in the dirty map. An expunged entry in the + // clean map must be unexpunged and added to the dirty map before a new value + // can be stored to it. + // + // If the dirty map is nil, the next write to the map will initialize it by + // making a shallow copy of the clean map, omitting stale entries. + dirty map[interface{}]*entry + + // misses counts the number of loads since the read map was last updated that + // needed to lock mu to determine whether the key was present. + // + // Once enough misses have occurred to cover the cost of copying the dirty + // map, the dirty map will be promoted to the read map (in the unamended + // state) and the next store to the map will make a new dirty copy. + misses int +} + +// readOnly is an immutable struct stored atomically in the Map.read field. +type readOnly struct { + m map[interface{}]*entry + amended bool // true if the dirty map contains some key not in m. +} + +// expunged is an arbitrary pointer that marks entries which have been deleted +// from the dirty map. +var expunged = unsafe.Pointer(new(interface{})) + +// An entry is a slot in the map corresponding to a particular key. +type entry struct { + // p points to the interface{} value stored for the entry. + // + // If p == nil, the entry has been deleted and m.dirty == nil. + // + // If p == expunged, the entry has been deleted, m.dirty != nil, and the entry + // is missing from m.dirty. + // + // Otherwise, the entry is valid and recorded in m.read.m[key] and, if m.dirty + // != nil, in m.dirty[key]. + // + // An entry can be deleted by atomic replacement with nil: when m.dirty is + // next created, it will atomically replace nil with expunged and leave + // m.dirty[key] unset. + // + // An entry's associated value can be updated by atomic replacement, provided + // p != expunged. If p == expunged, an entry's associated value can be updated + // only after first setting m.dirty[key] = e so that lookups using the dirty + // map find the entry. + p unsafe.Pointer // *interface{} +} + +func newEntry(i interface{}) *entry { + return &entry{p: unsafe.Pointer(&i)} +} + +// Load returns the value stored in the map for a key, or nil if no +// value is present. +// The ok result indicates whether value was found in the map. +func (m *Map) Load(key interface{}) (value interface{}, ok bool) { + read, _ := m.read.Load().(readOnly) + e, ok := read.m[key] + if !ok && read.amended { + m.mu.Lock() + // Avoid reporting a spurious miss if m.dirty got promoted while we were + // blocked on m.mu. (If further loads of the same key will not miss, it's + // not worth copying the dirty map for this key.) + read, _ = m.read.Load().(readOnly) + e, ok = read.m[key] + if !ok && read.amended { + e, ok = m.dirty[key] + // Regardless of whether the entry was present, record a miss: this key + // will take the slow path until the dirty map is promoted to the read + // map. + m.missLocked() + } + m.mu.Unlock() + } + if !ok { + return nil, false + } + return e.load() +} + +func (e *entry) load() (value interface{}, ok bool) { + p := atomic.LoadPointer(&e.p) + if p == nil || p == expunged { + return nil, false + } + return *(*interface{})(p), true +} + +// Store sets the value for a key. +func (m *Map) Store(key, value interface{}) { + read, _ := m.read.Load().(readOnly) + if e, ok := read.m[key]; ok && e.tryStore(&value) { + return + } + + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + if e, ok := read.m[key]; ok { + if e.unexpungeLocked() { + // The entry was previously expunged, which implies that there is a + // non-nil dirty map and this entry is not in it. + m.dirty[key] = e + } + e.storeLocked(&value) + } else if e, ok := m.dirty[key]; ok { + e.storeLocked(&value) + } else { + if !read.amended { + // We're adding the first new key to the dirty map. + // Make sure it is allocated and mark the read-only map as incomplete. + m.dirtyLocked() + m.read.Store(readOnly{m: read.m, amended: true}) + } + m.dirty[key] = newEntry(value) + } + m.mu.Unlock() +} + +// tryStore stores a value if the entry has not been expunged. +// +// If the entry is expunged, tryStore returns false and leaves the entry +// unchanged. +func (e *entry) tryStore(i *interface{}) bool { + p := atomic.LoadPointer(&e.p) + if p == expunged { + return false + } + for { + if atomic.CompareAndSwapPointer(&e.p, p, unsafe.Pointer(i)) { + return true + } + p = atomic.LoadPointer(&e.p) + if p == expunged { + return false + } + } +} + +// unexpungeLocked ensures that the entry is not marked as expunged. +// +// If the entry was previously expunged, it must be added to the dirty map +// before m.mu is unlocked. +func (e *entry) unexpungeLocked() (wasExpunged bool) { + return atomic.CompareAndSwapPointer(&e.p, expunged, nil) +} + +// storeLocked unconditionally stores a value to the entry. +// +// The entry must be known not to be expunged. +func (e *entry) storeLocked(i *interface{}) { + atomic.StorePointer(&e.p, unsafe.Pointer(i)) +} + +// LoadOrStore returns the existing value for the key if present. +// Otherwise, it stores and returns the given value. +// The loaded result is true if the value was loaded, false if stored. +func (m *Map) LoadOrStore(key, value interface{}) (actual interface{}, loaded bool) { + // Avoid locking if it's a clean hit. + read, _ := m.read.Load().(readOnly) + if e, ok := read.m[key]; ok { + actual, loaded, ok := e.tryLoadOrStore(value) + if ok { + return actual, loaded + } + } + + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + if e, ok := read.m[key]; ok { + if e.unexpungeLocked() { + m.dirty[key] = e + } + actual, loaded, _ = e.tryLoadOrStore(value) + } else if e, ok := m.dirty[key]; ok { + actual, loaded, _ = e.tryLoadOrStore(value) + m.missLocked() + } else { + if !read.amended { + // We're adding the first new key to the dirty map. + // Make sure it is allocated and mark the read-only map as incomplete. + m.dirtyLocked() + m.read.Store(readOnly{m: read.m, amended: true}) + } + m.dirty[key] = newEntry(value) + actual, loaded = value, false + } + m.mu.Unlock() + + return actual, loaded +} + +// tryLoadOrStore atomically loads or stores a value if the entry is not +// expunged. +// +// If the entry is expunged, tryLoadOrStore leaves the entry unchanged and +// returns with ok==false. +func (e *entry) tryLoadOrStore(i interface{}) (actual interface{}, loaded, ok bool) { + p := atomic.LoadPointer(&e.p) + if p == expunged { + return nil, false, false + } + if p != nil { + return *(*interface{})(p), true, true + } + + // Copy the interface after the first load to make this method more amenable + // to escape analysis: if we hit the "load" path or the entry is expunged, we + // shouldn't bother heap-allocating. + ic := i + for { + if atomic.CompareAndSwapPointer(&e.p, nil, unsafe.Pointer(&ic)) { + return i, false, true + } + p = atomic.LoadPointer(&e.p) + if p == expunged { + return nil, false, false + } + if p != nil { + return *(*interface{})(p), true, true + } + } +} + +// Delete deletes the value for a key. +func (m *Map) Delete(key interface{}) { + read, _ := m.read.Load().(readOnly) + e, ok := read.m[key] + if !ok && read.amended { + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + e, ok = read.m[key] + if !ok && read.amended { + delete(m.dirty, key) + } + m.mu.Unlock() + } + if ok { + e.delete() + } +} + +func (e *entry) delete() (hadValue bool) { + for { + p := atomic.LoadPointer(&e.p) + if p == nil || p == expunged { + return false + } + if atomic.CompareAndSwapPointer(&e.p, p, nil) { + return true + } + } +} + +// Range calls f sequentially for each key and value present in the map. +// If f returns false, range stops the iteration. +// +// Range does not necessarily correspond to any consistent snapshot of the Map's +// contents: no key will be visited more than once, but if the value for any key +// is stored or deleted concurrently, Range may reflect any mapping for that key +// from any point during the Range call. +// +// Range may be O(N) with the number of elements in the map even if f returns +// false after a constant number of calls. +func (m *Map) Range(f func(key, value interface{}) bool) { + // We need to be able to iterate over all of the keys that were already + // present at the start of the call to Range. + // If read.amended is false, then read.m satisfies that property without + // requiring us to hold m.mu for a long time. + read, _ := m.read.Load().(readOnly) + if read.amended { + // m.dirty contains keys not in read.m. Fortunately, Range is already O(N) + // (assuming the caller does not break out early), so a call to Range + // amortizes an entire copy of the map: we can promote the dirty copy + // immediately! + m.mu.Lock() + read, _ = m.read.Load().(readOnly) + if read.amended { + read = readOnly{m: m.dirty} + m.read.Store(read) + m.dirty = nil + m.misses = 0 + } + m.mu.Unlock() + } + + for k, e := range read.m { + v, ok := e.load() + if !ok { + continue + } + if !f(k, v) { + break + } + } +} + +func (m *Map) missLocked() { + m.misses++ + if m.misses < len(m.dirty) { + return + } + m.read.Store(readOnly{m: m.dirty}) + m.dirty = nil + m.misses = 0 +} + +func (m *Map) dirtyLocked() { + if m.dirty != nil { + return + } + + read, _ := m.read.Load().(readOnly) + m.dirty = make(map[interface{}]*entry, len(read.m)) + for k, e := range read.m { + if !e.tryExpungeLocked() { + m.dirty[k] = e + } + } +} + +func (e *entry) tryExpungeLocked() (isExpunged bool) { + p := atomic.LoadPointer(&e.p) + for p == nil { + if atomic.CompareAndSwapPointer(&e.p, nil, expunged) { + return true + } + p = atomic.LoadPointer(&e.p) + } + return p == expunged +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 607f193a38..9e1cbfc664 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -501,6 +501,12 @@ "revision": "b4690f45fa1cafc47b1c280c2e75116efe40cc13", "revisionTime": "2017-02-15T08:41:58Z" }, + { + "checksumSHA1": "4TEYFKrAUuwBMqExjQBsnf/CgjQ=", + "path": "golang.org/x/sync/syncmap", + "revision": "f52d1811a62927559de87708c8913c1650ce4f26", + "revisionTime": "2017-05-17T20:25:26Z" + }, { "checksumSHA1": "rTPzsn0jeqfgnQR0OsMKR8JRy5Y=", "path": "golang.org/x/sys/unix", diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index 1e10b1d76f..d09246f699 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -26,8 +26,6 @@ import ( "sync" "time" - "sync/atomic" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -35,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/syndtr/goleveldb/leveldb/errors" "golang.org/x/crypto/pbkdf2" + "golang.org/x/sync/syncmap" set "gopkg.in/fatih/set.v0" ) @@ -46,38 +45,12 @@ type Statistics struct { totalMessagesCleared int } -type settingType byte -type settingsMap map[settingType]interface{} - const ( - minPowIdx settingType = iota // Minimal PoW required by the whisper node - maxMsgSizeIdx settingType = iota // Maximal message length allowed by the whisper node - OverflowIdx settingType = iota // Indicator of message queue overflow + minPowIdx = iota // Minimal PoW required by the whisper node + maxMsgSizeIdx = iota // Maximal message length allowed by the whisper node + overflowIdx = iota // Indicator of message queue overflow ) -type settingsVault struct { - vaultMu sync.Mutex - vault atomic.Value -} - -func (s *settingsVault) get(idx settingType) interface{} { - m := s.vault.Load().(settingsMap) - return m[idx] -} - -func (s *settingsVault) store(idx settingType, val interface{}) { - s.vaultMu.Lock() - defer s.vaultMu.Unlock() - - m1 := s.vault.Load().(settingsMap) - m2 := make(settingsMap) - for k, v := range m1 { - m2[k] = v - } - m2[idx] = val - s.vault.Store(m2) -} - // Whisper represents a dark communication interface through the Ethereum // network, using its very own P2P communication layer. type Whisper struct { @@ -99,7 +72,7 @@ type Whisper struct { p2pMsgQueue chan *Envelope // Message queue for peer-to-peer messages (not to be forwarded any further) quit chan struct{} // Channel used for graceful exit - settings settingsVault // holds configuration settings that can be dynamically changed + settings syncmap.Map // holds configuration settings that can be dynamically changed statsMu sync.Mutex // guard stats stats Statistics // Statistics of whisper node @@ -126,10 +99,9 @@ func New(cfg *Config) *Whisper { whisper.filters = NewFilters(whisper) - whisper.settings.vault.Store(make(settingsMap)) - whisper.settings.store(minPowIdx, cfg.MinimumAcceptedPOW) - whisper.settings.store(maxMsgSizeIdx, cfg.MaxMessageSize) - whisper.settings.store(OverflowIdx, false) + whisper.settings.Store(minPowIdx, cfg.MinimumAcceptedPOW) + whisper.settings.Store(maxMsgSizeIdx, cfg.MaxMessageSize) + whisper.settings.Store(overflowIdx, false) // p2p whisper sub protocol handler whisper.protocol = p2p.Protocol{ @@ -150,17 +122,20 @@ func New(cfg *Config) *Whisper { } func (w *Whisper) MinPow() float64 { - return w.settings.get(minPowIdx).(float64) + val, _ := w.settings.Load(minPowIdx) + return val.(float64) } // MaxMessageSize returns the maximum accepted message size. func (w *Whisper) MaxMessageSize() uint32 { - return w.settings.get(maxMsgSizeIdx).(uint32) + val, _ := w.settings.Load(maxMsgSizeIdx) + return val.(uint32) } // Overflow returns an indication if the message queue is full. func (w *Whisper) Overflow() bool { - return w.settings.get(OverflowIdx).(bool) + val, _ := w.settings.Load(overflowIdx) + return val.(bool) } // APIs returns the RPC descriptors the Whisper implementation offers @@ -196,7 +171,7 @@ func (w *Whisper) SetMaxMessageSize(size uint32) error { if size > MaxMessageSize { return fmt.Errorf("message size too large [%d>%d]", size, MaxMessageSize) } - w.settings.store(maxMsgSizeIdx, uint32(size)) + w.settings.Store(maxMsgSizeIdx, uint32(size)) return nil } @@ -205,7 +180,7 @@ func (w *Whisper) SetMinimumPoW(val float64) error { if val <= 0.0 { return fmt.Errorf("invalid PoW: %f", val) } - w.settings.store(minPowIdx, val) + w.settings.Store(minPowIdx, val) return nil } @@ -679,12 +654,12 @@ func (w *Whisper) checkOverflow() { if queueSize == messageQueueLimit { if !w.Overflow() { - w.settings.store(OverflowIdx, true) + w.settings.Store(overflowIdx, true) log.Warn("message queue overflow") } } else if queueSize <= messageQueueLimit/2 { if w.Overflow() { - w.settings.store(OverflowIdx, false) + w.settings.Store(overflowIdx, false) log.Warn("message queue overflow fixed (back to normal)") } } From c62d5422bb24cfa53ee480330f9d355f68c071a4 Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Wed, 21 Jun 2017 12:58:00 +0200 Subject: [PATCH 09/43] whisper: use hexutil.UnmarshalFixedText for topic parsing --- cmd/geth/consolecmd.go | 2 +- whisper/whisperv5/topic.go | 22 ++++++-------------- whisper/whisperv5/topic_test.go | 37 ++++++++++++++++++--------------- 3 files changed, 27 insertions(+), 34 deletions(-) diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index f8a923aac1..2bb452d736 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -35,7 +35,7 @@ var ( Action: utils.MigrateFlags(localConsole), Name: "console", Usage: "Start an interactive JavaScript environment", - Flags: append(append(nodeFlags, rpcFlags...), consoleFlags...), + Flags: append(append(append(nodeFlags, rpcFlags...), consoleFlags...), whisperFlags...), Category: "CONSOLE COMMANDS", Description: ` The Geth console is an interactive shell for the JavaScript runtime environment diff --git a/whisper/whisperv5/topic.go b/whisper/whisperv5/topic.go index 5fea2f28bd..d1996c4604 100644 --- a/whisper/whisperv5/topic.go +++ b/whisper/whisperv5/topic.go @@ -19,9 +19,6 @@ package whisperv5 import ( - "encoding/json" - "fmt" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" ) @@ -47,19 +44,12 @@ func (topic *TopicType) String() string { return string(common.ToHex(topic[:])) } -func (t *TopicType) MarshalJSON() ([]byte, error) { - return json.Marshal(hexutil.Bytes(t[:])) +// MarshalText returns the hex representation of t. +func (t TopicType) MarshalText() ([]byte, error) { + return hexutil.Bytes(t[:]).MarshalText() } -// UnmarshalJSON parses a hex representation to a topic. -func (t *TopicType) UnmarshalJSON(input []byte) error { - var data hexutil.Bytes - if err := json.Unmarshal(input, &data); err != nil { - return err - } - if len(data) != TopicLength { - return fmt.Errorf("unmarshalJSON failed: topic must be exactly %d bytes(%d)", TopicLength, len(input)) - } - *t = BytesToTopic(data) - return nil +// UnmarshalText parses a hex representation to a topic. +func (t *TopicType) UnmarshalText(input []byte) error { + return hexutil.UnmarshalFixedText("Topic", input, t[:]) } diff --git a/whisper/whisperv5/topic_test.go b/whisper/whisperv5/topic_test.go index 63593963b1..54bbeaf85e 100644 --- a/whisper/whisperv5/topic_test.go +++ b/whisper/whisperv5/topic_test.go @@ -16,7 +16,10 @@ package whisperv5 -import "testing" +import ( + "encoding/json" + "testing" +) var topicStringTests = []struct { topic TopicType @@ -53,15 +56,6 @@ var bytesToTopicTests = []struct { {topic: TopicType{0x00, 0x00, 0x00, 0x00}, data: nil}, } -func TestBytesToTopic(t *testing.T) { - for i, tst := range bytesToTopicTests { - top := BytesToTopic(tst.data) - if top != tst.topic { - t.Fatalf("failed test %d: have %v, want %v.", i, t, tst.topic) - } - } -} - var unmarshalTestsGood = []struct { topic TopicType data []byte @@ -94,14 +88,23 @@ var unmarshalTestsUgly = []struct { {topic: TopicType{0x01, 0x00, 0x00, 0x00}, data: []byte(`"0x00000001"`)}, } +func TestBytesToTopic(t *testing.T) { + for i, tst := range bytesToTopicTests { + top := BytesToTopic(tst.data) + if top != tst.topic { + t.Fatalf("failed test %d: have %v, want %v.", i, t, tst.topic) + } + } +} + func TestUnmarshalTestsGood(t *testing.T) { for i, tst := range unmarshalTestsGood { var top TopicType - err := top.UnmarshalJSON(tst.data) + err := json.Unmarshal(tst.data, &top) if err != nil { - t.Fatalf("failed test %d. input: %v. err: %v", i, tst.data, err) + t.Errorf("failed test %d. input: %v. err: %v", i, tst.data, err) } else if top != tst.topic { - t.Fatalf("failed test %d: have %v, want %v.", i, t, tst.topic) + t.Errorf("failed test %d: have %v, want %v.", i, t, tst.topic) } } } @@ -110,7 +113,7 @@ func TestUnmarshalTestsBad(t *testing.T) { // in this test UnmarshalJSON() is supposed to fail for i, tst := range unmarshalTestsBad { var top TopicType - err := top.UnmarshalJSON(tst.data) + err := json.Unmarshal(tst.data, &top) if err == nil { t.Fatalf("failed test %d. input: %v.", i, tst.data) } @@ -121,11 +124,11 @@ func TestUnmarshalTestsUgly(t *testing.T) { // in this test UnmarshalJSON() is NOT supposed to fail, but result should be wrong for i, tst := range unmarshalTestsUgly { var top TopicType - err := top.UnmarshalJSON(tst.data) + err := json.Unmarshal(tst.data, &top) if err != nil { - t.Fatalf("failed test %d. input: %v.", i, tst.data) + t.Errorf("failed test %d. input: %v.", i, tst.data) } else if top == tst.topic { - t.Fatalf("failed test %d: have %v, want %v.", i, top, tst.topic) + t.Errorf("failed test %d: have %v, want %v.", i, top, tst.topic) } } } From beb708e6d76d83748c59dd92b2cb6d12194b7900 Mon Sep 17 00:00:00 2001 From: "mark.lin" Date: Fri, 23 Jun 2017 10:39:38 +0800 Subject: [PATCH 10/43] core: remove unused events --- core/events.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/core/events.go b/core/events.go index bc7c9ddcb3..6f404f612b 100644 --- a/core/events.go +++ b/core/events.go @@ -24,9 +24,6 @@ import ( // TxPreEvent is posted when a transaction enters the transaction pool. type TxPreEvent struct{ Tx *types.Transaction } -// TxPostEvent is posted when a transaction has been processed. -type TxPostEvent struct{ Tx *types.Transaction } - // PendingLogsEvent is posted pre mining and notifies of pending logs. type PendingLogsEvent struct { Logs []*types.Log @@ -54,17 +51,4 @@ type ChainSideEvent struct { Block *types.Block } -type PendingBlockEvent struct { - Block *types.Block - Logs []*types.Log -} - -type ChainUncleEvent struct { - Block *types.Block -} - type ChainHeadEvent struct{ Block *types.Block } - -// Mining operation events -type StartMining struct{} -type TopMining struct{} From b6e99deee9517c7571ab31c950b0108e94486a2e Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Fri, 23 Jun 2017 09:18:02 +0200 Subject: [PATCH 11/43] whisper: renamed Info#Message to Info#Messages --- whisper/whisperv5/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/whisper/whisperv5/api.go b/whisper/whisperv5/api.go index f3d25f781d..5b84b99eb4 100644 --- a/whisper/whisperv5/api.go +++ b/whisper/whisperv5/api.go @@ -94,7 +94,7 @@ func (api *PublicWhisperAPI) Version(ctx context.Context) string { // Info contains diagnostic information. type Info struct { Memory int `json:"memory"` // Memory size of the floating messages in bytes. - Message int `json:"message"` // Number of floating messages. + Messages int `json:"messages"` // Number of floating messages. MinPow float64 `json:"minPow"` // Minimal accepted PoW MaxMessageSize uint32 `json:"maxMessageSize"` // Maximum accepted message size } @@ -104,7 +104,7 @@ func (api *PublicWhisperAPI) Info(ctx context.Context) Info { stats := api.w.Stats() return Info{ Memory: stats.memoryUsed, - Message: len(api.w.messageQueue) + len(api.w.p2pMsgQueue), + Messages: len(api.w.messageQueue) + len(api.w.p2pMsgQueue), MinPow: api.w.MinPow(), MaxMessageSize: api.w.MaxMessageSize(), } From 78c04c920d07e65864f781fb5c6e5d77435d7aa0 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 23 Jun 2017 11:28:43 +0200 Subject: [PATCH 12/43] params, VERSION: 1.6.7 unstable --- VERSION | 2 +- params/version.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index ec70f75560..400084b1bf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.6.6 +1.6.7 diff --git a/params/version.go b/params/version.go index 196d9987e5..d54c69b711 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 6 // Minor version component of the current release - VersionPatch = 6 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 6 // Minor version component of the current release + VersionPatch = 7 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 5603eb911602c4527c6633c1ce2c3de65f8100fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 23 Jun 2017 13:34:21 +0300 Subject: [PATCH 13/43] cmd/puppeth: fix key reuse during faucet deploys --- cmd/puppeth/wizard_faucet.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go index 66ec98c73b..51c4e2f7f4 100644 --- a/cmd/puppeth/wizard_faucet.go +++ b/cmd/puppeth/wizard_faucet.go @@ -165,8 +165,7 @@ func (w *wizard) deployFaucet() { } // Load up the credential needed to release funds if infos.node.keyJSON != "" { - var key keystore.Key - if err := json.Unmarshal([]byte(infos.node.keyJSON), &key); err != nil { + if key, err := keystore.DecryptKey([]byte(infos.node.keyJSON), infos.node.keyPass); err != nil { infos.node.keyJSON, infos.node.keyPass = "", "" } else { fmt.Println() From caa00b7e6d0dcb0429ef83ae993a2656abde9b60 Mon Sep 17 00:00:00 2001 From: aron Date: Sun, 25 Jun 2017 16:10:54 +0200 Subject: [PATCH 14/43] swarm/storage: remove panic on invalid chunk --- swarm/storage/dbstore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 30925a919f..31ff5b64e6 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -399,7 +399,7 @@ func (s *DbStore) Get(key Key) (chunk *Chunk, err error) { hash := hasher.Sum(nil) if !bytes.Equal(hash, key) { s.delete(index.Idx, getIndexKey(key)) - panic("Invalid Chunk in Database. Please repair with command: 'swarm cleandb'") + log.Warn("Invalid Chunk in Database. Please repair with command: 'swarm cleandb'") } chunk = &Chunk{ From ef8d4711d531de716425c83c3ef56fb6b198108f Mon Sep 17 00:00:00 2001 From: "G. Kay Lee" Date: Mon, 26 Jun 2017 17:17:06 +0800 Subject: [PATCH 15/43] Update README.md (#14701) README: change heading to "Go Ethereum" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index eb707c2172..8d4f37f5ea 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -## Ethereum Go +## Go Ethereum Official golang implementation of the Ethereum protocol. From 3c7338d6c86cee5769d16c49bba3ef57b722e971 Mon Sep 17 00:00:00 2001 From: Aron Date: Mon, 26 Jun 2017 11:19:53 +0200 Subject: [PATCH 16/43] Makefile: add make swarm command (#14698) * Makefile: add make swarm command * Makefile: minor code formatting polishes --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2b5d84f286..b6e2ddd5e5 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # with Go source code. If you know what GOPATH is then you probably # don't need to bother with make. -.PHONY: geth android ios geth-cross evm all test clean +.PHONY: geth android ios geth-cross swarm evm all test clean .PHONY: geth-linux geth-linux-386 geth-linux-amd64 geth-linux-mips64 geth-linux-mips64le .PHONY: geth-linux-arm geth-linux-arm-5 geth-linux-arm-6 geth-linux-arm-7 geth-linux-arm64 .PHONY: geth-darwin geth-darwin-386 geth-darwin-amd64 @@ -16,6 +16,11 @@ geth: @echo "Done building." @echo "Run \"$(GOBIN)/geth\" to launch geth." +swarm: + build/env.sh go run build/ci.go install ./cmd/swarm + @echo "Done building." + @echo "Run \"$(GOBIN)/swarm\" to launch swarm." + evm: build/env.sh go run build/ci.go install ./cmd/evm @echo "Done building." From 413dc1d2651541dddb242716416a0427b0a656da Mon Sep 17 00:00:00 2001 From: bloonfield Date: Mon, 26 Jun 2017 17:26:22 +0800 Subject: [PATCH 17/43] rpc: fix closure problem in batch processing (#14688) Demo of the issue: https://play.golang.org/p/EeTLFfppqC --- rpc/http.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rpc/http.go b/rpc/http.go index 6bab02ab67..4143e2a8dc 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -103,8 +103,8 @@ func (c *Client) sendBatchHTTP(ctx context.Context, op *requestOp, msgs []*jsonr if err := json.NewDecoder(respBody).Decode(&respmsgs); err != nil { return err } - for _, respmsg := range respmsgs { - op.resp <- &respmsg + for i := 0; i < len(respmsgs); i++ { + op.resp <- &respmsgs[i] } return nil } From ea1d1825a8509b3353c535c9444861e15471942a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 26 Jun 2017 13:40:43 +0300 Subject: [PATCH 18/43] cmd/geth: fix whisper flag group capitalization --- cmd/geth/usage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index bb14de747b..022c8e899e 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -188,7 +188,7 @@ var AppHelpFlagGroups = []flagGroup{ }, debug.Flags...), }, { - Name: "Whisper (EXPERIMENTAL)", + Name: "WHISPER (EXPERIMENTAL)", Flags: whisperFlags, }, { From f3359d5e585eef09648c7a06a6d3911704f61e73 Mon Sep 17 00:00:00 2001 From: Lewis Marshall Date: Sun, 18 Jun 2017 00:25:39 +0200 Subject: [PATCH 19/43] cmd/swarm: Support using Mainnet for resolving ENS names Signed-off-by: Lewis Marshall --- cmd/swarm/main.go | 105 +++++++++++++++++++++++++++++++++++++++---- contracts/ens/ens.go | 5 +++ swarm/api/config.go | 9 ++-- swarm/swarm.go | 8 ++-- 4 files changed, 109 insertions(+), 18 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 71149c310c..f0680c487e 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -17,21 +17,25 @@ package main import ( + "context" "crypto/ecdsa" "fmt" "io/ioutil" + "math/big" "os" "os/signal" "runtime" "strconv" "strings" "syscall" + "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console" + "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/internal/debug" @@ -40,6 +44,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" "gopkg.in/urfave/cli.v1" @@ -87,15 +92,27 @@ var ( Name: "swap", Usage: "Swarm SWAP enabled (default false)", } + SwarmSwapAPIFlag = cli.StringFlag{ + Name: "swap-api", + Usage: "URL of the Ethereum API provider to use to settle SWAP payments", + } SwarmSyncEnabledFlag = cli.BoolTFlag{ Name: "sync", Usage: "Swarm Syncing enabled (default true)", } EthAPIFlag = cli.StringFlag{ Name: "ethapi", - Usage: "URL of the Ethereum API provider", + Usage: "DEPRECATED: please use --ens-api and --swap-api", + } + EnsAPIFlag = cli.StringFlag{ + Name: "ens-api", + Usage: "URL of the Ethereum API provider to use for ENS record lookups", Value: node.DefaultIPCEndpoint("geth"), } + EnsAddrFlag = cli.StringFlag{ + Name: "ens-addr", + Usage: "ENS contract address (default is detected as testnet or mainnet using --ens-api)", + } SwarmApiFlag = cli.StringFlag{ Name: "bzzapi", Usage: "Swarm HTTP endpoint", @@ -250,8 +267,11 @@ Cleans database of corrupted entries. // bzzd-specific flags CorsStringFlag, EthAPIFlag, + EnsAPIFlag, + EnsAddrFlag, SwarmConfigPathFlag, SwarmSwapEnabledFlag, + SwarmSwapAPIFlag, SwarmSyncEnabledFlag, SwarmListenAddrFlag, SwarmPortFlag, @@ -333,6 +353,38 @@ func bzzd(ctx *cli.Context) error { return nil } +// detectEnsAddr determines the ENS contract address by getting both the +// version and genesis hash using the client and matching them to either +// mainnet or testnet addresses +func detectEnsAddr(client *rpc.Client) (common.Address, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var version string + if err := client.CallContext(ctx, &version, "net_version"); err != nil { + return common.Address{}, err + } + + block, err := ethclient.NewClient(client).BlockByNumber(ctx, big.NewInt(0)) + if err != nil { + return common.Address{}, err + } + + switch { + + case version == "1" && block.Hash() == params.MainNetGenesisHash: + log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress) + return ens.MainNetAddress, nil + + case version == "3" && block.Hash() == params.TestNetGenesisHash: + log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress) + return ens.TestNetAddress, nil + + default: + return common.Address{}, fmt.Errorf("unknown version and genesis hash: %s %s", version, block.Hash()) + } +} + func registerBzzService(ctx *cli.Context, stack *node.Node) { prvkey := getAccount(ctx, stack) @@ -357,19 +409,56 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name) ethapi := ctx.GlobalString(EthAPIFlag.Name) + if ethapi != "" { + log.Warn("DEPRECATED: --ethapi is deprecated and will be removed in a future version, please use --ens-api and --swap-api") + } + + swapapi := ctx.GlobalString(SwarmSwapAPIFlag.Name) + if swapEnabled && swapapi == "" { + utils.Fatalf("SWAP is enabled but --swap-api is not set") + } + + ensapi := ctx.GlobalString(EnsAPIFlag.Name) + // use the deprecated --ethapi if --ens-api is not set + if ensapi == "" { + ensapi = ethapi + } + ensAddr := ctx.GlobalString(EnsAddrFlag.Name) + cors := ctx.GlobalString(CorsStringFlag.Name) boot := func(ctx *node.ServiceContext) (node.Service, error) { - var client *ethclient.Client - if len(ethapi) > 0 { - client, err = ethclient.Dial(ethapi) + var swapClient *ethclient.Client + if swapapi != "" { + log.Info("connecting to SWAP API", "url", swapapi) + swapClient, err = ethclient.Dial(swapapi) if err != nil { - utils.Fatalf("Can't connect: %v", err) + return nil, fmt.Errorf("error connecting to SWAP API %s: %s", swapapi, err) } - } else { - swapEnabled = false } - return swarm.NewSwarm(ctx, client, bzzconfig, swapEnabled, syncEnabled, cors) + + var ensClient *ethclient.Client + if ensapi != "" { + log.Info("connecting to ENS API", "url", ensapi) + client, err := rpc.Dial(ensapi) + if err != nil { + return nil, fmt.Errorf("error connecting to ENS API %s: %s", ensapi, err) + } + ensClient = ethclient.NewClient(client) + + if ensAddr != "" { + bzzconfig.EnsRoot = common.HexToAddress(ensAddr) + } else { + ensAddr, err := detectEnsAddr(client) + if err == nil { + bzzconfig.EnsRoot = ensAddr + } else { + log.Warn(fmt.Sprintf("could not determine ENS contract address, using default %s", bzzconfig.EnsRoot), "err", err) + } + } + } + + return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, swapEnabled, syncEnabled, cors) } if err := stack.Register(boot); err != nil { utils.Fatalf("Failed to register the Swarm service: %v", err) diff --git a/contracts/ens/ens.go b/contracts/ens/ens.go index 83ca9b63b1..c292a1714a 100644 --- a/contracts/ens/ens.go +++ b/contracts/ens/ens.go @@ -29,6 +29,11 @@ import ( "github.com/ethereum/go-ethereum/crypto" ) +var ( + MainNetAddress = common.HexToAddress("0x314159265dD8dbb310642f98f50C066173C1259b") + TestNetAddress = common.HexToAddress("0x112234455c3a32fd11230c42e7bccd4a84e02010") +) + // swarm domain name registry and resolver type ENS struct { *contract.ENSSession diff --git a/swarm/api/config.go b/swarm/api/config.go index 647c153eda..d8d25b1c82 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -25,6 +25,7 @@ import ( "path/filepath" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/services/swap" @@ -36,10 +37,6 @@ const ( DefaultHTTPPort = "8500" ) -var ( - ensRootAddress = common.HexToAddress("0x112234455c3a32fd11230c42e7bccd4a84e02010") -) - // separate bzz directories // allow several bzz nodes running in parallel type Config struct { @@ -84,7 +81,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n Swap: swap.DefaultSwapParams(contract, prvKey), PublicKey: pubkeyhex, BzzKey: keyhex, - EnsRoot: ensRootAddress, + EnsRoot: ens.TestNetAddress, NetworkId: networkId, } data, err = ioutil.ReadFile(confpath) @@ -129,7 +126,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n self.Swap.SetKey(prvKey) if (self.EnsRoot == common.Address{}) { - self.EnsRoot = ensRootAddress + self.EnsRoot = ens.TestNetAddress } return diff --git a/swarm/swarm.go b/swarm/swarm.go index 4f93a30b79..8304908433 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -76,7 +76,7 @@ func (self *Swarm) API() *SwarmAPI { // creates a new swarm service instance // implements node.Service -func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.Config, swapEnabled, syncEnabled bool, cors string) (self *Swarm, err error) { +func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string) (self *Swarm, err error) { if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) { return nil, fmt.Errorf("empty public key") } @@ -136,10 +136,10 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. // set up high level api transactOpts := bind.NewKeyedTransactor(self.privateKey) - if backend == (*ethclient.Client)(nil) { - log.Warn("No ENS, please specify non-empty --ethapi to use domain name resolution") + if ensClient == nil { + log.Warn("No ENS, please specify non-empty --ens-api to use domain name resolution") } else { - self.dns, err = ens.NewENS(transactOpts, config.EnsRoot, self.backend) + self.dns, err = ens.NewENS(transactOpts, config.EnsRoot, ensClient) if err != nil { return nil, err } From cf611c50b9bd7a7c410a49699e0c4e718b4a7177 Mon Sep 17 00:00:00 2001 From: Addy Yeow Date: Tue, 27 Jun 2017 17:11:41 +1000 Subject: [PATCH 20/43] build: fix devel golang detection on debian/ubuntu (#14711) --- build/ci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/ci.go b/build/ci.go index 47b1dc780b..6a52077d45 100644 --- a/build/ci.go +++ b/build/ci.go @@ -175,7 +175,7 @@ func doInstall(cmdline []string) { // Check Go version. People regularly open issues about compilation // failure with outdated Go. This should save them the trouble. - if runtime.Version() < "go1.7" && !strings.HasPrefix(runtime.Version(), "devel") { + if runtime.Version() < "go1.7" && !strings.Contains(runtime.Version(), "devel") { log.Println("You have Go version", runtime.Version()) log.Println("go-ethereum requires at least Go version 1.7 and cannot") log.Println("be compiled with an earlier version. Please upgrade your Go installation.") From 5421a08d2f924558056c8a382017ec37f56010e9 Mon Sep 17 00:00:00 2001 From: RJ Catalano Date: Tue, 27 Jun 2017 03:05:33 -0500 Subject: [PATCH 21/43] accounts/abi: reorganizing package with small fixes (#14610) * accounts/abi: reorganizing package and some notes and a quick correction of name. Signed-off-by: RJ Catalano get rid of some imports Signed-off-by: RJ Catalano * accounts/abi: move file names Signed-off-by: RJ Catalano * accounts/abi: fix boolean decode function Signed-off-by: RJ Catalano * accounts/abi: fix for the array set and for creating a bool Signed-off-by: RJ Catalano * accounts/abi: be very very very correct Signed-off-by: RJ Catalano * accounts/abi: fix up error message and variable names Signed-off-by: RJ Catalano * accounts/abi: take out unnecessary argument in pack method Signed-off-by: RJ Catalano * accounts/abi: add bool unpack test and add a panic to readBool function Signed-off-by: RJ Catalano * accounts/abi: fix panic message Signed-off-by: RJ Catalano * accounts/abi: change from panic to basic error Signed-off-by: RJ Catalano * accounts/abi: fix nil to false Signed-off-by: RJ Catalano * accounts/abi: fill out type regex tests and fill with the correct type for integers Signed-off-by: RJ Catalano * accounts/abi: move packNumbers into pack.go. Signed-off-by: RJ Catalano * accounts/abi: separation of the testing suite into appropriately named files. Signed-off-by: RJ Catalano * account/abi: change to hex string tests. Signed-off-by: RJ Catalano * account/abi: fix up rest of tests to hex Signed-off-by: RJ Catalano * accounts/abi: declare bool at the package level Signed-off-by: RJ Catalano * accounts/abi: use errors package in the error file. Signed-off-by: RJ Catalano * accounts/abi: fix ugly hack and fix error type declaration. Signed-off-by: RJ Catalano --- accounts/abi/abi.go | 197 +------ accounts/abi/abi_test.go | 802 --------------------------- accounts/abi/error.go | 5 + accounts/abi/method.go | 2 +- accounts/abi/numbers.go | 13 - accounts/abi/numbers_test.go | 38 -- accounts/abi/{packing.go => pack.go} | 15 +- accounts/abi/pack_test.go | 441 +++++++++++++++ accounts/abi/reflect.go | 20 +- accounts/abi/type.go | 8 +- accounts/abi/type_test.go | 115 +++- accounts/abi/unpack.go | 235 ++++++++ accounts/abi/unpack_test.go | 681 +++++++++++++++++++++++ 13 files changed, 1487 insertions(+), 1085 deletions(-) rename accounts/abi/{packing.go => pack.go} (80%) create mode 100644 accounts/abi/pack_test.go create mode 100644 accounts/abi/unpack.go create mode 100644 accounts/abi/unpack_test.go diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index 3d1010229c..2a06d474b8 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -17,11 +17,9 @@ package abi import ( - "encoding/binary" "encoding/json" "fmt" "io" - "math/big" "reflect" "strings" @@ -67,7 +65,7 @@ func (abi ABI) Pack(name string, args ...interface{}) ([]byte, error) { } method = m } - arguments, err := method.pack(method, args...) + arguments, err := method.pack(args...) if err != nil { return nil, err } @@ -78,199 +76,6 @@ func (abi ABI) Pack(name string, args ...interface{}) ([]byte, error) { return append(method.Id(), arguments...), nil } -// toGoSliceType parses the input and casts it to the proper slice defined by the ABI -// argument in T. -func toGoSlice(i int, t Argument, output []byte) (interface{}, error) { - index := i * 32 - // The slice must, at very least be large enough for the index+32 which is exactly the size required - // for the [offset in output, size of offset]. - if index+32 > len(output) { - return nil, fmt.Errorf("abi: cannot marshal in to go slice: insufficient size output %d require %d", len(output), index+32) - } - elem := t.Type.Elem - - // first we need to create a slice of the type - var refSlice reflect.Value - switch elem.T { - case IntTy, UintTy, BoolTy: - // create a new reference slice matching the element type - switch t.Type.Kind { - case reflect.Bool: - refSlice = reflect.ValueOf([]bool(nil)) - case reflect.Uint8: - refSlice = reflect.ValueOf([]uint8(nil)) - case reflect.Uint16: - refSlice = reflect.ValueOf([]uint16(nil)) - case reflect.Uint32: - refSlice = reflect.ValueOf([]uint32(nil)) - case reflect.Uint64: - refSlice = reflect.ValueOf([]uint64(nil)) - case reflect.Int8: - refSlice = reflect.ValueOf([]int8(nil)) - case reflect.Int16: - refSlice = reflect.ValueOf([]int16(nil)) - case reflect.Int32: - refSlice = reflect.ValueOf([]int32(nil)) - case reflect.Int64: - refSlice = reflect.ValueOf([]int64(nil)) - default: - refSlice = reflect.ValueOf([]*big.Int(nil)) - } - case AddressTy: // address must be of slice Address - refSlice = reflect.ValueOf([]common.Address(nil)) - case HashTy: // hash must be of slice hash - refSlice = reflect.ValueOf([]common.Hash(nil)) - case FixedBytesTy: - refSlice = reflect.ValueOf([][]byte(nil)) - default: // no other types are supported - return nil, fmt.Errorf("abi: unsupported slice type %v", elem.T) - } - - var slice []byte - var size int - var offset int - if t.Type.IsSlice { - // get the offset which determines the start of this array ... - offset = int(binary.BigEndian.Uint64(output[index+24 : index+32])) - if offset+32 > len(output) { - return nil, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32) - } - - slice = output[offset:] - // ... starting with the size of the array in elements ... - size = int(binary.BigEndian.Uint64(slice[24:32])) - slice = slice[32:] - // ... and make sure that we've at the very least the amount of bytes - // available in the buffer. - if size*32 > len(slice) { - return nil, fmt.Errorf("abi: cannot marshal in to go slice: insufficient size output %d require %d", len(output), offset+32+size*32) - } - - // reslice to match the required size - slice = slice[:size*32] - } else if t.Type.IsArray { - //get the number of elements in the array - size = t.Type.SliceSize - - //check to make sure array size matches up - if index+32*size > len(output) { - return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), index+32*size) - } - //slice is there for a fixed amount of times - slice = output[index : index+size*32] - } - - for i := 0; i < size; i++ { - var ( - inter interface{} // interface type - returnOutput = slice[i*32 : i*32+32] // the return output - ) - // set inter to the correct type (cast) - switch elem.T { - case IntTy, UintTy: - inter = readInteger(t.Type.Kind, returnOutput) - case BoolTy: - inter = !allZero(returnOutput) - case AddressTy: - inter = common.BytesToAddress(returnOutput) - case HashTy: - inter = common.BytesToHash(returnOutput) - case FixedBytesTy: - inter = returnOutput - } - // append the item to our reflect slice - refSlice = reflect.Append(refSlice, reflect.ValueOf(inter)) - } - - // return the interface - return refSlice.Interface(), nil -} - -func readInteger(kind reflect.Kind, b []byte) interface{} { - switch kind { - case reflect.Uint8: - return uint8(b[len(b)-1]) - case reflect.Uint16: - return binary.BigEndian.Uint16(b[len(b)-2:]) - case reflect.Uint32: - return binary.BigEndian.Uint32(b[len(b)-4:]) - case reflect.Uint64: - return binary.BigEndian.Uint64(b[len(b)-8:]) - case reflect.Int8: - return int8(b[len(b)-1]) - case reflect.Int16: - return int16(binary.BigEndian.Uint16(b[len(b)-2:])) - case reflect.Int32: - return int32(binary.BigEndian.Uint32(b[len(b)-4:])) - case reflect.Int64: - return int64(binary.BigEndian.Uint64(b[len(b)-8:])) - default: - return new(big.Int).SetBytes(b) - } -} - -func allZero(b []byte) bool { - for _, byte := range b { - if byte != 0 { - return false - } - } - return true -} - -// toGoType parses the input and casts it to the proper type defined by the ABI -// argument in T. -func toGoType(i int, t Argument, output []byte) (interface{}, error) { - // we need to treat slices differently - if (t.Type.IsSlice || t.Type.IsArray) && t.Type.T != BytesTy && t.Type.T != StringTy && t.Type.T != FixedBytesTy && t.Type.T != FunctionTy { - return toGoSlice(i, t, output) - } - - index := i * 32 - if index+32 > len(output) { - return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), index+32) - } - - // Parse the given index output and check whether we need to read - // a different offset and length based on the type (i.e. string, bytes) - var returnOutput []byte - switch t.Type.T { - case StringTy, BytesTy: // variable arrays are written at the end of the return bytes - // parse offset from which we should start reading - offset := int(binary.BigEndian.Uint64(output[index+24 : index+32])) - if offset+32 > len(output) { - return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32) - } - // parse the size up until we should be reading - size := int(binary.BigEndian.Uint64(output[offset+24 : offset+32])) - if offset+32+size > len(output) { - return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+size) - } - - // get the bytes for this return value - returnOutput = output[offset+32 : offset+32+size] - default: - returnOutput = output[index : index+32] - } - - // convert the bytes to whatever is specified by the ABI. - switch t.Type.T { - case IntTy, UintTy: - return readInteger(t.Type.Kind, returnOutput), nil - case BoolTy: - return !allZero(returnOutput), nil - case AddressTy: - return common.BytesToAddress(returnOutput), nil - case HashTy: - return common.BytesToHash(returnOutput), nil - case BytesTy, FixedBytesTy, FunctionTy: - return returnOutput, nil - case StringTy: - return string(returnOutput), nil - } - return nil, fmt.Errorf("abi: unknown type %v", t.Type.T) -} - // these variable are used to determine certain types during type assertion for // assignment. var ( diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index a45bd6cc0f..a3aa9446ef 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -48,412 +48,6 @@ func pad(input []byte, size int, left bool) []byte { return common.RightPadBytes(input, size) } -func TestTypeCheck(t *testing.T) { - for i, test := range []struct { - typ string - input interface{} - err string - }{ - {"uint", big.NewInt(1), ""}, - {"int", big.NewInt(1), ""}, - {"uint30", big.NewInt(1), ""}, - {"uint30", uint8(1), "abi: cannot use uint8 as type ptr as argument"}, - {"uint16", uint16(1), ""}, - {"uint16", uint8(1), "abi: cannot use uint8 as type uint16 as argument"}, - {"uint16[]", []uint16{1, 2, 3}, ""}, - {"uint16[]", [3]uint16{1, 2, 3}, ""}, - {"uint16[]", []uint32{1, 2, 3}, "abi: cannot use []uint32 as type []uint16 as argument"}, - {"uint16[3]", [3]uint32{1, 2, 3}, "abi: cannot use [3]uint32 as type [3]uint16 as argument"}, - {"uint16[3]", [4]uint16{1, 2, 3}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"}, - {"uint16[3]", []uint16{1, 2, 3}, ""}, - {"uint16[3]", []uint16{1, 2, 3, 4}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"}, - {"address[]", []common.Address{{1}}, ""}, - {"address[1]", []common.Address{{1}}, ""}, - {"address[1]", [1]common.Address{{1}}, ""}, - {"address[2]", [1]common.Address{{1}}, "abi: cannot use [1]array as type [2]array as argument"}, - {"bytes32", [32]byte{}, ""}, - {"bytes32", [33]byte{}, "abi: cannot use [33]uint8 as type [32]uint8 as argument"}, - {"bytes32", common.Hash{1}, ""}, - {"bytes31", [31]byte{}, ""}, - {"bytes31", [32]byte{}, "abi: cannot use [32]uint8 as type [31]uint8 as argument"}, - {"bytes", []byte{0, 1}, ""}, - {"bytes", [2]byte{0, 1}, ""}, - {"bytes", common.Hash{1}, ""}, - {"string", "hello world", ""}, - {"bytes32[]", [][32]byte{{}}, ""}, - {"function", [24]byte{}, ""}, - } { - typ, err := NewType(test.typ) - if err != nil { - t.Fatal("unexpected parse error:", err) - } - - err = typeCheck(typ, reflect.ValueOf(test.input)) - if err != nil && len(test.err) == 0 { - t.Errorf("%d failed. Expected no err but got: %v", i, err) - continue - } - if err == nil && len(test.err) != 0 { - t.Errorf("%d failed. Expected err: %v but got none", i, test.err) - continue - } - - if err != nil && len(test.err) != 0 && err.Error() != test.err { - t.Errorf("%d failed. Expected err: '%v' got err: '%v'", i, test.err, err) - } - } -} - -func TestSimpleMethodUnpack(t *testing.T) { - for i, test := range []struct { - def string // definition of the **output** ABI params - marshalledOutput []byte // evm return data - expectedOut interface{} // the expected output - outVar string // the output variable (e.g. uint32, *big.Int, etc) - err string // empty or error if expected - }{ - { - `[ { "type": "uint32" } ]`, - pad([]byte{1}, 32, true), - uint32(1), - "uint32", - "", - }, - { - `[ { "type": "uint32" } ]`, - pad([]byte{1}, 32, true), - nil, - "uint16", - "abi: cannot unmarshal uint32 in to uint16", - }, - { - `[ { "type": "uint17" } ]`, - pad([]byte{1}, 32, true), - nil, - "uint16", - "abi: cannot unmarshal *big.Int in to uint16", - }, - { - `[ { "type": "uint17" } ]`, - pad([]byte{1}, 32, true), - big.NewInt(1), - "*big.Int", - "", - }, - - { - `[ { "type": "int32" } ]`, - pad([]byte{1}, 32, true), - int32(1), - "int32", - "", - }, - { - `[ { "type": "int32" } ]`, - pad([]byte{1}, 32, true), - nil, - "int16", - "abi: cannot unmarshal int32 in to int16", - }, - { - `[ { "type": "int17" } ]`, - pad([]byte{1}, 32, true), - nil, - "int16", - "abi: cannot unmarshal *big.Int in to int16", - }, - { - `[ { "type": "int17" } ]`, - pad([]byte{1}, 32, true), - big.NewInt(1), - "*big.Int", - "", - }, - - { - `[ { "type": "address" } ]`, - pad(pad([]byte{1}, 20, false), 32, true), - common.Address{1}, - "address", - "", - }, - { - `[ { "type": "bytes32" } ]`, - pad([]byte{1}, 32, false), - pad([]byte{1}, 32, false), - "bytes", - "", - }, - { - `[ { "type": "bytes32" } ]`, - pad([]byte{1}, 32, false), - pad([]byte{1}, 32, false), - "hash", - "", - }, - { - `[ { "type": "bytes32" } ]`, - pad([]byte{1}, 32, false), - pad([]byte{1}, 32, false), - "interface", - "", - }, - { - `[ { "type": "function" } ]`, - pad([]byte{1}, 32, false), - [24]byte{1}, - "function", - "", - }, - } { - abiDefinition := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def) - abi, err := JSON(strings.NewReader(abiDefinition)) - if err != nil { - t.Errorf("%d failed. %v", i, err) - continue - } - - var outvar interface{} - switch test.outVar { - case "uint8": - var v uint8 - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "uint16": - var v uint16 - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "uint32": - var v uint32 - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "uint64": - var v uint64 - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "int8": - var v int8 - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "int16": - var v int16 - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "int32": - var v int32 - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "int64": - var v int64 - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "*big.Int": - var v *big.Int - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "address": - var v common.Address - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "bytes": - var v []byte - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "hash": - var v common.Hash - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "function": - var v [24]byte - err = abi.Unpack(&v, "method", test.marshalledOutput) - outvar = v - case "interface": - err = abi.Unpack(&outvar, "method", test.marshalledOutput) - default: - t.Errorf("unsupported type '%v' please add it to the switch statement in this test", test.outVar) - continue - } - - if err != nil && len(test.err) == 0 { - t.Errorf("%d failed. Expected no err but got: %v", i, err) - continue - } - if err == nil && len(test.err) != 0 { - t.Errorf("%d failed. Expected err: %v but got none", i, test.err) - continue - } - if err != nil && len(test.err) != 0 && err.Error() != test.err { - t.Errorf("%d failed. Expected err: '%v' got err: '%v'", i, test.err, err) - continue - } - - if err == nil { - // bit of an ugly hack for hash type but I don't feel like finding a proper solution - if test.outVar == "hash" { - tmp := outvar.(common.Hash) // without assignment it's unaddressable - outvar = tmp[:] - } - - if !reflect.DeepEqual(test.expectedOut, outvar) { - t.Errorf("%d failed. Output error: expected %v, got %v", i, test.expectedOut, outvar) - } - } - } -} - -func TestUnpackSetInterfaceSlice(t *testing.T) { - var ( - var1 = new(uint8) - var2 = new(uint8) - ) - out := []interface{}{var1, var2} - abi, err := JSON(strings.NewReader(`[{"type":"function", "name":"ints", "outputs":[{"type":"uint8"}, {"type":"uint8"}]}]`)) - if err != nil { - t.Fatal(err) - } - marshalledReturn := append(pad([]byte{1}, 32, true), pad([]byte{2}, 32, true)...) - err = abi.Unpack(&out, "ints", marshalledReturn) - if err != nil { - t.Fatal(err) - } - if *var1 != 1 { - t.Error("expected var1 to be 1, got", *var1) - } - if *var2 != 2 { - t.Error("expected var2 to be 2, got", *var2) - } - - out = []interface{}{var1} - err = abi.Unpack(&out, "ints", marshalledReturn) - - expErr := "abi: cannot marshal in to slices of unequal size (require: 2, got: 1)" - if err == nil || err.Error() != expErr { - t.Error("expected err:", expErr, "Got:", err) - } -} - -func TestUnpackSetInterfaceArrayOutput(t *testing.T) { - var ( - var1 = new([1]uint32) - var2 = new([1]uint32) - ) - out := []interface{}{var1, var2} - abi, err := JSON(strings.NewReader(`[{"type":"function", "name":"ints", "outputs":[{"type":"uint32[1]"}, {"type":"uint32[1]"}]}]`)) - if err != nil { - t.Fatal(err) - } - marshalledReturn := append(pad([]byte{1}, 32, true), pad([]byte{2}, 32, true)...) - err = abi.Unpack(&out, "ints", marshalledReturn) - if err != nil { - t.Fatal(err) - } - - if *var1 != [1]uint32{1} { - t.Error("expected var1 to be [1], got", *var1) - } - if *var2 != [1]uint32{2} { - t.Error("expected var2 to be [2], got", *var2) - } -} - -func TestPack(t *testing.T) { - for i, test := range []struct { - typ string - - input interface{} - output []byte - }{ - {"uint16", uint16(2), pad([]byte{2}, 32, true)}, - {"uint16[]", []uint16{1, 2}, formatSliceOutput([]byte{1}, []byte{2})}, - {"bytes20", [20]byte{1}, pad([]byte{1}, 32, false)}, - {"uint256[]", []*big.Int{big.NewInt(1), big.NewInt(2)}, formatSliceOutput([]byte{1}, []byte{2})}, - {"address[]", []common.Address{{1}, {2}}, formatSliceOutput(pad([]byte{1}, 20, false), pad([]byte{2}, 20, false))}, - {"bytes32[]", []common.Hash{{1}, {2}}, formatSliceOutput(pad([]byte{1}, 32, false), pad([]byte{2}, 32, false))}, - {"function", [24]byte{1}, pad([]byte{1}, 32, false)}, - } { - typ, err := NewType(test.typ) - if err != nil { - t.Fatal("unexpected parse error:", err) - } - - output, err := typ.pack(reflect.ValueOf(test.input)) - if err != nil { - t.Fatal("unexpected pack error:", err) - } - - if !bytes.Equal(output, test.output) { - t.Errorf("%d failed. Expected bytes: '%x' Got: '%x'", i, test.output, output) - } - } -} - -func TestMethodPack(t *testing.T) { - abi, err := JSON(strings.NewReader(jsondata2)) - if err != nil { - t.Fatal(err) - } - - sig := abi.Methods["slice"].Id() - sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) - sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) - - packed, err := abi.Pack("slice", []uint32{1, 2}) - if err != nil { - t.Error(err) - } - - if !bytes.Equal(packed, sig) { - t.Errorf("expected %x got %x", sig, packed) - } - - var addrA, addrB = common.Address{1}, common.Address{2} - sig = abi.Methods["sliceAddress"].Id() - sig = append(sig, common.LeftPadBytes([]byte{32}, 32)...) - sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) - sig = append(sig, common.LeftPadBytes(addrA[:], 32)...) - sig = append(sig, common.LeftPadBytes(addrB[:], 32)...) - - packed, err = abi.Pack("sliceAddress", []common.Address{addrA, addrB}) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(packed, sig) { - t.Errorf("expected %x got %x", sig, packed) - } - - var addrC, addrD = common.Address{3}, common.Address{4} - sig = abi.Methods["sliceMultiAddress"].Id() - sig = append(sig, common.LeftPadBytes([]byte{64}, 32)...) - sig = append(sig, common.LeftPadBytes([]byte{160}, 32)...) - sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) - sig = append(sig, common.LeftPadBytes(addrA[:], 32)...) - sig = append(sig, common.LeftPadBytes(addrB[:], 32)...) - sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) - sig = append(sig, common.LeftPadBytes(addrC[:], 32)...) - sig = append(sig, common.LeftPadBytes(addrD[:], 32)...) - - packed, err = abi.Pack("sliceMultiAddress", []common.Address{addrA, addrB}, []common.Address{addrC, addrD}) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(packed, sig) { - t.Errorf("expected %x got %x", sig, packed) - } - - sig = abi.Methods["slice256"].Id() - sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) - sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) - - packed, err = abi.Pack("slice256", []*big.Int{big.NewInt(1), big.NewInt(2)}) - if err != nil { - t.Error(err) - } - - if !bytes.Equal(packed, sig) { - t.Errorf("expected %x got %x", sig, packed) - } -} - const jsondata = ` [ { "type" : "function", "name" : "balance", "constant" : true }, @@ -843,399 +437,3 @@ func TestBareEvents(t *testing.T) { } } } - -func TestMultiReturnWithStruct(t *testing.T) { - const definition = `[ - { "name" : "multi", "constant" : false, "outputs": [ { "name": "Int", "type": "uint256" }, { "name": "String", "type": "string" } ] }]` - - abi, err := JSON(strings.NewReader(definition)) - if err != nil { - t.Fatal(err) - } - - // using buff to make the code readable - buff := new(bytes.Buffer) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000005")) - stringOut := "hello" - buff.Write(common.RightPadBytes([]byte(stringOut), 32)) - - var inter struct { - Int *big.Int - String string - } - err = abi.Unpack(&inter, "multi", buff.Bytes()) - if err != nil { - t.Error(err) - } - - if inter.Int == nil || inter.Int.Cmp(big.NewInt(1)) != 0 { - t.Error("expected Int to be 1 got", inter.Int) - } - - if inter.String != stringOut { - t.Error("expected String to be", stringOut, "got", inter.String) - } - - var reversed struct { - String string - Int *big.Int - } - - err = abi.Unpack(&reversed, "multi", buff.Bytes()) - if err != nil { - t.Error(err) - } - - if reversed.Int == nil || reversed.Int.Cmp(big.NewInt(1)) != 0 { - t.Error("expected Int to be 1 got", reversed.Int) - } - - if reversed.String != stringOut { - t.Error("expected String to be", stringOut, "got", reversed.String) - } -} - -func TestMultiReturnWithSlice(t *testing.T) { - const definition = `[ - { "name" : "multi", "constant" : false, "outputs": [ { "name": "Int", "type": "uint256" }, { "name": "String", "type": "string" } ] }]` - - abi, err := JSON(strings.NewReader(definition)) - if err != nil { - t.Fatal(err) - } - - // using buff to make the code readable - buff := new(bytes.Buffer) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000005")) - stringOut := "hello" - buff.Write(common.RightPadBytes([]byte(stringOut), 32)) - - var inter []interface{} - err = abi.Unpack(&inter, "multi", buff.Bytes()) - if err != nil { - t.Error(err) - } - - if len(inter) != 2 { - t.Fatal("expected 2 results got", len(inter)) - } - - if num, ok := inter[0].(*big.Int); !ok || num.Cmp(big.NewInt(1)) != 0 { - t.Error("expected index 0 to be 1 got", num) - } - - if str, ok := inter[1].(string); !ok || str != stringOut { - t.Error("expected index 1 to be", stringOut, "got", str) - } -} - -func TestMarshalArrays(t *testing.T) { - const definition = `[ - { "name" : "bytes32", "constant" : false, "outputs": [ { "type": "bytes32" } ] }, - { "name" : "bytes10", "constant" : false, "outputs": [ { "type": "bytes10" } ] } - ]` - - abi, err := JSON(strings.NewReader(definition)) - if err != nil { - t.Fatal(err) - } - - output := common.LeftPadBytes([]byte{1}, 32) - - var bytes10 [10]byte - err = abi.Unpack(&bytes10, "bytes32", output) - if err == nil || err.Error() != "abi: cannot unmarshal src (len=32) in to dst (len=10)" { - t.Error("expected error or bytes32 not be assignable to bytes10:", err) - } - - var bytes32 [32]byte - err = abi.Unpack(&bytes32, "bytes32", output) - if err != nil { - t.Error("didn't expect error:", err) - } - if !bytes.Equal(bytes32[:], output) { - t.Error("expected bytes32[31] to be 1 got", bytes32[31]) - } - - type ( - B10 [10]byte - B32 [32]byte - ) - - var b10 B10 - err = abi.Unpack(&b10, "bytes32", output) - if err == nil || err.Error() != "abi: cannot unmarshal src (len=32) in to dst (len=10)" { - t.Error("expected error or bytes32 not be assignable to bytes10:", err) - } - - var b32 B32 - err = abi.Unpack(&b32, "bytes32", output) - if err != nil { - t.Error("didn't expect error:", err) - } - if !bytes.Equal(b32[:], output) { - t.Error("expected bytes32[31] to be 1 got", bytes32[31]) - } - - output[10] = 1 - var shortAssignLong [32]byte - err = abi.Unpack(&shortAssignLong, "bytes10", output) - if err != nil { - t.Error("didn't expect error:", err) - } - if !bytes.Equal(output, shortAssignLong[:]) { - t.Errorf("expected %x to be %x", shortAssignLong, output) - } -} - -func TestUnmarshal(t *testing.T) { - const definition = `[ - { "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] }, - { "name" : "bool", "constant" : false, "outputs": [ { "type": "bool" } ] }, - { "name" : "bytes", "constant" : false, "outputs": [ { "type": "bytes" } ] }, - { "name" : "fixed", "constant" : false, "outputs": [ { "type": "bytes32" } ] }, - { "name" : "multi", "constant" : false, "outputs": [ { "type": "bytes" }, { "type": "bytes" } ] }, - { "name" : "intArraySingle", "constant" : false, "outputs": [ { "type": "uint256[3]" } ] }, - { "name" : "addressSliceSingle", "constant" : false, "outputs": [ { "type": "address[]" } ] }, - { "name" : "addressSliceDouble", "constant" : false, "outputs": [ { "name": "a", "type": "address[]" }, { "name": "b", "type": "address[]" } ] }, - { "name" : "mixedBytes", "constant" : true, "outputs": [ { "name": "a", "type": "bytes" }, { "name": "b", "type": "bytes32" } ] }]` - - abi, err := JSON(strings.NewReader(definition)) - if err != nil { - t.Fatal(err) - } - buff := new(bytes.Buffer) - - // marshal int - var Int *big.Int - err = abi.Unpack(&Int, "int", common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) - if err != nil { - t.Error(err) - } - - if Int == nil || Int.Cmp(big.NewInt(1)) != 0 { - t.Error("expected Int to be 1 got", Int) - } - - // marshal bool - var Bool bool - err = abi.Unpack(&Bool, "bool", common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) - if err != nil { - t.Error(err) - } - - if !Bool { - t.Error("expected Bool to be true") - } - - // marshal dynamic bytes max length 32 - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - bytesOut := common.RightPadBytes([]byte("hello"), 32) - buff.Write(bytesOut) - - var Bytes []byte - err = abi.Unpack(&Bytes, "bytes", buff.Bytes()) - if err != nil { - t.Error(err) - } - - if !bytes.Equal(Bytes, bytesOut) { - t.Errorf("expected %x got %x", bytesOut, Bytes) - } - - // marshall dynamic bytes max length 64 - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) - bytesOut = common.RightPadBytes([]byte("hello"), 64) - buff.Write(bytesOut) - - err = abi.Unpack(&Bytes, "bytes", buff.Bytes()) - if err != nil { - t.Error(err) - } - - if !bytes.Equal(Bytes, bytesOut) { - t.Errorf("expected %x got %x", bytesOut, Bytes) - } - - // marshall dynamic bytes max length 63 - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000003f")) - bytesOut = common.RightPadBytes([]byte("hello"), 63) - buff.Write(bytesOut) - - err = abi.Unpack(&Bytes, "bytes", buff.Bytes()) - if err != nil { - t.Error(err) - } - - if !bytes.Equal(Bytes, bytesOut) { - t.Errorf("expected %x got %x", bytesOut, Bytes) - } - - // marshal dynamic bytes output empty - err = abi.Unpack(&Bytes, "bytes", nil) - if err == nil { - t.Error("expected error") - } - - // marshal dynamic bytes length 5 - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000005")) - buff.Write(common.RightPadBytes([]byte("hello"), 32)) - - err = abi.Unpack(&Bytes, "bytes", buff.Bytes()) - if err != nil { - t.Error(err) - } - - if !bytes.Equal(Bytes, []byte("hello")) { - t.Errorf("expected %x got %x", bytesOut, Bytes) - } - - // marshal dynamic bytes length 5 - buff.Reset() - buff.Write(common.RightPadBytes([]byte("hello"), 32)) - - var hash common.Hash - err = abi.Unpack(&hash, "fixed", buff.Bytes()) - if err != nil { - t.Error(err) - } - - helloHash := common.BytesToHash(common.RightPadBytes([]byte("hello"), 32)) - if hash != helloHash { - t.Errorf("Expected %x to equal %x", hash, helloHash) - } - - // marshal error - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - err = abi.Unpack(&Bytes, "bytes", buff.Bytes()) - if err == nil { - t.Error("expected error") - } - - err = abi.Unpack(&Bytes, "multi", make([]byte, 64)) - if err == nil { - t.Error("expected error") - } - - // marshal mixed bytes - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) - fixed := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001") - buff.Write(fixed) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) - bytesOut = common.RightPadBytes([]byte("hello"), 32) - buff.Write(bytesOut) - - var out []interface{} - err = abi.Unpack(&out, "mixedBytes", buff.Bytes()) - if err != nil { - t.Fatal("didn't expect error:", err) - } - - if !bytes.Equal(bytesOut, out[0].([]byte)) { - t.Errorf("expected %x, got %x", bytesOut, out[0]) - } - - if !bytes.Equal(fixed, out[1].([]byte)) { - t.Errorf("expected %x, got %x", fixed, out[1]) - } - - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003")) - // marshal int array - var intArray [3]*big.Int - err = abi.Unpack(&intArray, "intArraySingle", buff.Bytes()) - if err != nil { - t.Error(err) - } - var testAgainstIntArray [3]*big.Int - testAgainstIntArray[0] = big.NewInt(1) - testAgainstIntArray[1] = big.NewInt(2) - testAgainstIntArray[2] = big.NewInt(3) - - for i, Int := range intArray { - if Int.Cmp(testAgainstIntArray[i]) != 0 { - t.Errorf("expected %v, got %v", testAgainstIntArray[i], Int) - } - } - // marshal address slice - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) // offset - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // size - buff.Write(common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000")) - - var outAddr []common.Address - err = abi.Unpack(&outAddr, "addressSliceSingle", buff.Bytes()) - if err != nil { - t.Fatal("didn't expect error:", err) - } - - if len(outAddr) != 1 { - t.Fatal("expected 1 item, got", len(outAddr)) - } - - if outAddr[0] != (common.Address{1}) { - t.Errorf("expected %x, got %x", common.Address{1}, outAddr[0]) - } - - // marshal multiple address slice - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) // offset - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000080")) // offset - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // size - buff.Write(common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000")) - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // size - buff.Write(common.Hex2Bytes("0000000000000000000000000200000000000000000000000000000000000000")) - buff.Write(common.Hex2Bytes("0000000000000000000000000300000000000000000000000000000000000000")) - - var outAddrStruct struct { - A []common.Address - B []common.Address - } - err = abi.Unpack(&outAddrStruct, "addressSliceDouble", buff.Bytes()) - if err != nil { - t.Fatal("didn't expect error:", err) - } - - if len(outAddrStruct.A) != 1 { - t.Fatal("expected 1 item, got", len(outAddrStruct.A)) - } - - if outAddrStruct.A[0] != (common.Address{1}) { - t.Errorf("expected %x, got %x", common.Address{1}, outAddrStruct.A[0]) - } - - if len(outAddrStruct.B) != 2 { - t.Fatal("expected 1 item, got", len(outAddrStruct.B)) - } - - if outAddrStruct.B[0] != (common.Address{2}) { - t.Errorf("expected %x, got %x", common.Address{2}, outAddrStruct.B[0]) - } - if outAddrStruct.B[1] != (common.Address{3}) { - t.Errorf("expected %x, got %x", common.Address{3}, outAddrStruct.B[1]) - } - - // marshal invalid address slice - buff.Reset() - buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000100")) - - err = abi.Unpack(&outAddr, "addressSliceSingle", buff.Bytes()) - if err == nil { - t.Fatal("expected error:", err) - } -} diff --git a/accounts/abi/error.go b/accounts/abi/error.go index 67739c21dc..420acf4182 100644 --- a/accounts/abi/error.go +++ b/accounts/abi/error.go @@ -17,10 +17,15 @@ package abi import ( + "errors" "fmt" "reflect" ) +var ( + errBadBool = errors.New("abi: improperly encoded boolean value") +) + // formatSliceString formats the reflection kind with the given slice size // and returns a formatted string representation. func formatSliceString(kind reflect.Kind, sliceSize int) string { diff --git a/accounts/abi/method.go b/accounts/abi/method.go index d56f3bc3dd..62b3d29575 100644 --- a/accounts/abi/method.go +++ b/accounts/abi/method.go @@ -39,7 +39,7 @@ type Method struct { Outputs []Argument } -func (m Method) pack(method Method, args ...interface{}) ([]byte, error) { +func (method Method) pack(args ...interface{}) ([]byte, error) { // Make sure arguments match up and pack them if len(args) != len(method.Inputs) { return nil, fmt.Errorf("argument count mismatch: %d for %d", len(args), len(method.Inputs)) diff --git a/accounts/abi/numbers.go b/accounts/abi/numbers.go index 10afa6511f..5d3efff52e 100644 --- a/accounts/abi/numbers.go +++ b/accounts/abi/numbers.go @@ -62,19 +62,6 @@ func U256(n *big.Int) []byte { return math.PaddedBigBytes(math.U256(n), 32) } -// packNum packs the given number (using the reflect value) and will cast it to appropriate number representation -func packNum(value reflect.Value) []byte { - switch kind := value.Kind(); kind { - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return U256(new(big.Int).SetUint64(value.Uint())) - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return U256(big.NewInt(value.Int())) - case reflect.Ptr: - return U256(value.Interface().(*big.Int)) - } - return nil -} - // checks whether the given reflect value is signed. This also works for slices with a number type func isSigned(v reflect.Value) bool { switch v.Type() { diff --git a/accounts/abi/numbers_test.go b/accounts/abi/numbers_test.go index 44afe86478..b9ff5aef17 100644 --- a/accounts/abi/numbers_test.go +++ b/accounts/abi/numbers_test.go @@ -18,7 +18,6 @@ package abi import ( "bytes" - "math" "math/big" "reflect" "testing" @@ -34,43 +33,6 @@ func TestNumberTypes(t *testing.T) { } } -func TestPackNumber(t *testing.T) { - tests := []struct { - value reflect.Value - packed []byte - }{ - // Protocol limits - {reflect.ValueOf(0), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, - {reflect.ValueOf(1), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}}, - {reflect.ValueOf(-1), []byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}}, - - // Type corner cases - {reflect.ValueOf(uint8(math.MaxUint8)), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255}}, - {reflect.ValueOf(uint16(math.MaxUint16)), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255}}, - {reflect.ValueOf(uint32(math.MaxUint32)), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255}}, - {reflect.ValueOf(uint64(math.MaxUint64)), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, 255}}, - - {reflect.ValueOf(int8(math.MaxInt8)), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127}}, - {reflect.ValueOf(int16(math.MaxInt16)), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 255}}, - {reflect.ValueOf(int32(math.MaxInt32)), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 255, 255, 255}}, - {reflect.ValueOf(int64(math.MaxInt64)), []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 255, 255, 255, 255, 255, 255, 255}}, - - {reflect.ValueOf(int8(math.MinInt8)), []byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 128}}, - {reflect.ValueOf(int16(math.MinInt16)), []byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 128, 0}}, - {reflect.ValueOf(int32(math.MinInt32)), []byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 128, 0, 0, 0}}, - {reflect.ValueOf(int64(math.MinInt64)), []byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 128, 0, 0, 0, 0, 0, 0, 0}}, - } - for i, tt := range tests { - packed := packNum(tt.value) - if !bytes.Equal(packed, tt.packed) { - t.Errorf("test %d: pack mismatch: have %x, want %x", i, packed, tt.packed) - } - } - if packed := packNum(reflect.ValueOf("string")); packed != nil { - t.Errorf("expected 'string' to pack to nil. got %x instead", packed) - } -} - func TestSigned(t *testing.T) { if isSigned(reflect.ValueOf(uint(10))) { t.Error("signed") diff --git a/accounts/abi/packing.go b/accounts/abi/pack.go similarity index 80% rename from accounts/abi/packing.go rename to accounts/abi/pack.go index 1d7f85e2ba..4d8a3f0318 100644 --- a/accounts/abi/packing.go +++ b/accounts/abi/pack.go @@ -17,6 +17,7 @@ package abi import ( + "math/big" "reflect" "github.com/ethereum/go-ethereum/common" @@ -59,8 +60,20 @@ func packElement(t Type, reflectValue reflect.Value) []byte { if reflectValue.Kind() == reflect.Array { reflectValue = mustArrayToByteSlice(reflectValue) } - return common.RightPadBytes(reflectValue.Bytes(), 32) } panic("abi: fatal error") } + +// packNum packs the given number (using the reflect value) and will cast it to appropriate number representation +func packNum(value reflect.Value) []byte { + switch kind := value.Kind(); kind { + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return U256(new(big.Int).SetUint64(value.Uint())) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return U256(big.NewInt(value.Int())) + case reflect.Ptr: + return U256(value.Interface().(*big.Int)) + } + return nil +} diff --git a/accounts/abi/pack_test.go b/accounts/abi/pack_test.go new file mode 100644 index 0000000000..c6cfb56ea0 --- /dev/null +++ b/accounts/abi/pack_test.go @@ -0,0 +1,441 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package abi + +import ( + "bytes" + "math" + "math/big" + "reflect" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestPack(t *testing.T) { + for i, test := range []struct { + typ string + + input interface{} + output []byte + }{ + { + "uint8", + uint8(2), + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "uint8[]", + []uint8{1, 2}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "uint16", + uint16(2), + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "uint16[]", + []uint16{1, 2}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "uint32", + uint32(2), + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "uint32[]", + []uint32{1, 2}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "uint64", + uint64(2), + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "uint64[]", + []uint64{1, 2}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "uint256", + big.NewInt(2), + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "uint256[]", + []*big.Int{big.NewInt(1), big.NewInt(2)}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "int8", + int8(2), + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "int8[]", + []int8{1, 2}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "int16", + int16(2), + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "int16[]", + []int16{1, 2}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "int32", + int32(2), + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "int32[]", + []int32{1, 2}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "int64", + int64(2), + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "int64[]", + []int64{1, 2}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "int256", + big.NewInt(2), + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "int256[]", + []*big.Int{big.NewInt(1), big.NewInt(2)}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), + }, + { + "bytes1", + [1]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes2", + [2]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes3", + [3]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes4", + [4]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes5", + [5]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes6", + [6]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes7", + [7]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes8", + [8]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes9", + [9]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes10", + [10]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes11", + [11]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes12", + [12]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes13", + [13]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes14", + [14]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes15", + [15]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes16", + [16]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes17", + [17]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes18", + [18]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes19", + [19]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes20", + [20]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes21", + [21]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes22", + [22]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes23", + [23]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes24", + [24]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes24", + [24]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes25", + [25]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes26", + [26]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes27", + [27]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes28", + [28]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes29", + [29]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes30", + [30]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes31", + [31]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "bytes32", + [32]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "address[]", + []common.Address{{1}, {2}}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000"), + }, + { + "bytes32[]", + []common.Hash{{1}, {2}}, + common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000201000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000"), + }, + { + "function", + [24]byte{1}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + }, + { + "string", + "foobar", + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006666f6f6261720000000000000000000000000000000000000000000000000000"), + }, + } { + typ, err := NewType(test.typ) + if err != nil { + t.Fatal("unexpected parse error:", err) + } + + output, err := typ.pack(reflect.ValueOf(test.input)) + if err != nil { + t.Fatal("unexpected pack error:", err) + } + + if !bytes.Equal(output, test.output) { + t.Errorf("%d failed. Expected bytes: '%x' Got: '%x'", i, test.output, output) + } + } +} + +func TestMethodPack(t *testing.T) { + abi, err := JSON(strings.NewReader(jsondata2)) + if err != nil { + t.Fatal(err) + } + + sig := abi.Methods["slice"].Id() + sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + + packed, err := abi.Pack("slice", []uint32{1, 2}) + if err != nil { + t.Error(err) + } + + if !bytes.Equal(packed, sig) { + t.Errorf("expected %x got %x", sig, packed) + } + + var addrA, addrB = common.Address{1}, common.Address{2} + sig = abi.Methods["sliceAddress"].Id() + sig = append(sig, common.LeftPadBytes([]byte{32}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + sig = append(sig, common.LeftPadBytes(addrA[:], 32)...) + sig = append(sig, common.LeftPadBytes(addrB[:], 32)...) + + packed, err = abi.Pack("sliceAddress", []common.Address{addrA, addrB}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(packed, sig) { + t.Errorf("expected %x got %x", sig, packed) + } + + var addrC, addrD = common.Address{3}, common.Address{4} + sig = abi.Methods["sliceMultiAddress"].Id() + sig = append(sig, common.LeftPadBytes([]byte{64}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{160}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + sig = append(sig, common.LeftPadBytes(addrA[:], 32)...) + sig = append(sig, common.LeftPadBytes(addrB[:], 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + sig = append(sig, common.LeftPadBytes(addrC[:], 32)...) + sig = append(sig, common.LeftPadBytes(addrD[:], 32)...) + + packed, err = abi.Pack("sliceMultiAddress", []common.Address{addrA, addrB}, []common.Address{addrC, addrD}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(packed, sig) { + t.Errorf("expected %x got %x", sig, packed) + } + + sig = abi.Methods["slice256"].Id() + sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + + packed, err = abi.Pack("slice256", []*big.Int{big.NewInt(1), big.NewInt(2)}) + if err != nil { + t.Error(err) + } + + if !bytes.Equal(packed, sig) { + t.Errorf("expected %x got %x", sig, packed) + } +} + +func TestPackNumber(t *testing.T) { + tests := []struct { + value reflect.Value + packed []byte + }{ + // Protocol limits + {reflect.ValueOf(0), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")}, + {reflect.ValueOf(1), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")}, + {reflect.ValueOf(-1), common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")}, + + // Type corner cases + {reflect.ValueOf(uint8(math.MaxUint8)), common.Hex2Bytes("00000000000000000000000000000000000000000000000000000000000000ff")}, + {reflect.ValueOf(uint16(math.MaxUint16)), common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000ffff")}, + {reflect.ValueOf(uint32(math.MaxUint32)), common.Hex2Bytes("00000000000000000000000000000000000000000000000000000000ffffffff")}, + {reflect.ValueOf(uint64(math.MaxUint64)), common.Hex2Bytes("000000000000000000000000000000000000000000000000ffffffffffffffff")}, + + {reflect.ValueOf(int8(math.MaxInt8)), common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000007f")}, + {reflect.ValueOf(int16(math.MaxInt16)), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000007fff")}, + {reflect.ValueOf(int32(math.MaxInt32)), common.Hex2Bytes("000000000000000000000000000000000000000000000000000000007fffffff")}, + {reflect.ValueOf(int64(math.MaxInt64)), common.Hex2Bytes("0000000000000000000000000000000000000000000000007fffffffffffffff")}, + + {reflect.ValueOf(int8(math.MinInt8)), common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff80")}, + {reflect.ValueOf(int16(math.MinInt16)), common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8000")}, + {reflect.ValueOf(int32(math.MinInt32)), common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffff80000000")}, + {reflect.ValueOf(int64(math.MinInt64)), common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffff8000000000000000")}, + } + for i, tt := range tests { + packed := packNum(tt.value) + if !bytes.Equal(packed, tt.packed) { + t.Errorf("test %d: pack mismatch: have %x, want %x", i, packed, tt.packed) + } + } + if packed := packNum(reflect.ValueOf("string")); packed != nil { + t.Errorf("expected 'string' to pack to nil. got %x instead", packed) + } +} diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index 7970ba8ac1..8fa75df07f 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -32,30 +32,30 @@ func indirect(v reflect.Value) reflect.Value { // reflectIntKind returns the reflect using the given size and // unsignedness. -func reflectIntKind(unsigned bool, size int) reflect.Kind { +func reflectIntKindAndType(unsigned bool, size int) (reflect.Kind, reflect.Type) { switch size { case 8: if unsigned { - return reflect.Uint8 + return reflect.Uint8, uint8_t } - return reflect.Int8 + return reflect.Int8, int8_t case 16: if unsigned { - return reflect.Uint16 + return reflect.Uint16, uint16_t } - return reflect.Int16 + return reflect.Int16, int16_t case 32: if unsigned { - return reflect.Uint32 + return reflect.Uint32, uint32_t } - return reflect.Int32 + return reflect.Int32, int32_t case 64: if unsigned { - return reflect.Uint64 + return reflect.Uint64, uint64_t } - return reflect.Int64 + return reflect.Int64, int64_t } - return reflect.Ptr + return reflect.Ptr, big_t } // mustArrayToBytesSlice creates a new byte slice with the exact same size as value diff --git a/accounts/abi/type.go b/accounts/abi/type.go index f2832aef56..5f20babb3f 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -33,7 +33,7 @@ const ( FixedBytesTy BytesTy HashTy - FixedpointTy + FixedPointTy FunctionTy ) @@ -126,13 +126,11 @@ func NewType(t string) (typ Type, err error) { switch varType { case "int": - typ.Kind = reflectIntKind(false, varSize) - typ.Type = big_t + typ.Kind, typ.Type = reflectIntKindAndType(false, varSize) typ.Size = varSize typ.T = IntTy case "uint": - typ.Kind = reflectIntKind(true, varSize) - typ.Type = ubig_t + typ.Kind, typ.Type = reflectIntKindAndType(true, varSize) typ.Size = varSize typ.T = UintTy case "bool": diff --git a/accounts/abi/type_test.go b/accounts/abi/type_test.go index 1558064596..984a5bb4c1 100644 --- a/accounts/abi/type_test.go +++ b/accounts/abi/type_test.go @@ -17,8 +17,11 @@ package abi import ( + "math/big" "reflect" "testing" + + "github.com/ethereum/go-ethereum/common" ) // typeWithoutStringer is a alias for the Type type which simply doesn't implement @@ -31,26 +34,44 @@ func TestTypeRegexp(t *testing.T) { blob string kind Type }{ - {"int", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}}, - {"int8", Type{Kind: reflect.Int8, Type: big_t, Size: 8, T: IntTy, stringKind: "int8"}}, + {"bool", Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}}, + {"bool[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Bool, T: BoolTy, Elem: &Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}, stringKind: "bool[]"}}, + {"bool[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Bool, T: BoolTy, Elem: &Type{Kind: reflect.Bool, T: BoolTy, stringKind: "bool"}, stringKind: "bool[2]"}}, + {"int8", Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}}, + {"int16", Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}}, + {"int32", Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}}, + {"int64", Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}}, {"int256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}}, - {"int[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}}, - {"int[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}}, - {"int32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int32, Type: big_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: big_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}}, - {"int32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int32, Type: big_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: big_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}}, - {"uint", Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}}, - {"uint8", Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}}, - {"uint256", Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}}, - {"uint[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}}, - {"uint[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: ubig_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}}, - {"uint32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint32, Type: ubig_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: big_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}}, - {"uint32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint32, Type: ubig_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: big_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}}, - {"bytes", Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}}, - {"bytes32", Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}}, - {"bytes[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}}, - {"bytes[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[2]"}}, - {"bytes32[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[]"}}, - {"bytes32[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: ubig_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[2]"}}, + {"int8[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}}, + {"int8[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}}, + {"int16[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}}, + {"int16[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}}, + {"int32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}}, + {"int32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}}, + {"int64[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}}, + {"int64[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}}, + {"int256[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}}, + {"int256[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}}, + {"uint8", Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}}, + {"uint16", Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}}, + {"uint32", Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}}, + {"uint64", Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}}, + {"uint256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}}, + {"uint8[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}}, + {"uint8[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}}, + {"uint16[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}}, + {"uint16[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}}, + {"uint32[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}}, + {"uint32[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}}, + {"uint64[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}}, + {"uint64[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}}, + {"uint256[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}}, + {"uint256[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}}, + {"bytes32", Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}}, + {"bytes[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}}, + {"bytes[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsSlice: true, SliceSize: -1, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[2]"}}, + {"bytes32[]", Type{IsSlice: true, SliceSize: -1, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[]"}}, + {"bytes32[2]", Type{IsArray: true, SliceSize: 2, Elem: &Type{IsArray: true, SliceSize: 32, Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, T: FixedBytesTy, stringKind: "bytes32"}, stringKind: "bytes32[2]"}}, {"string", Type{Kind: reflect.String, Size: -1, T: StringTy, stringKind: "string"}}, {"string[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[]"}}, {"string[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[2]"}}, @@ -76,3 +97,59 @@ func TestTypeRegexp(t *testing.T) { } } } + +func TestTypeCheck(t *testing.T) { + for i, test := range []struct { + typ string + input interface{} + err string + }{ + {"uint", big.NewInt(1), ""}, + {"int", big.NewInt(1), ""}, + {"uint30", big.NewInt(1), ""}, + {"uint30", uint8(1), "abi: cannot use uint8 as type ptr as argument"}, + {"uint16", uint16(1), ""}, + {"uint16", uint8(1), "abi: cannot use uint8 as type uint16 as argument"}, + {"uint16[]", []uint16{1, 2, 3}, ""}, + {"uint16[]", [3]uint16{1, 2, 3}, ""}, + {"uint16[]", []uint32{1, 2, 3}, "abi: cannot use []uint32 as type []uint16 as argument"}, + {"uint16[3]", [3]uint32{1, 2, 3}, "abi: cannot use [3]uint32 as type [3]uint16 as argument"}, + {"uint16[3]", [4]uint16{1, 2, 3}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"}, + {"uint16[3]", []uint16{1, 2, 3}, ""}, + {"uint16[3]", []uint16{1, 2, 3, 4}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"}, + {"address[]", []common.Address{{1}}, ""}, + {"address[1]", []common.Address{{1}}, ""}, + {"address[1]", [1]common.Address{{1}}, ""}, + {"address[2]", [1]common.Address{{1}}, "abi: cannot use [1]array as type [2]array as argument"}, + {"bytes32", [32]byte{}, ""}, + {"bytes32", [33]byte{}, "abi: cannot use [33]uint8 as type [32]uint8 as argument"}, + {"bytes32", common.Hash{1}, ""}, + {"bytes31", [31]byte{}, ""}, + {"bytes31", [32]byte{}, "abi: cannot use [32]uint8 as type [31]uint8 as argument"}, + {"bytes", []byte{0, 1}, ""}, + {"bytes", [2]byte{0, 1}, ""}, + {"bytes", common.Hash{1}, ""}, + {"string", "hello world", ""}, + {"bytes32[]", [][32]byte{{}}, ""}, + {"function", [24]byte{}, ""}, + } { + typ, err := NewType(test.typ) + if err != nil { + t.Fatal("unexpected parse error:", err) + } + + err = typeCheck(typ, reflect.ValueOf(test.input)) + if err != nil && len(test.err) == 0 { + t.Errorf("%d failed. Expected no err but got: %v", i, err) + continue + } + if err == nil && len(test.err) != 0 { + t.Errorf("%d failed. Expected err: %v but got none", i, test.err) + continue + } + + if err != nil && len(test.err) != 0 && err.Error() != test.err { + t.Errorf("%d failed. Expected err: '%v' got err: '%v'", i, test.err, err) + } + } +} diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go new file mode 100644 index 0000000000..fc41c88ac7 --- /dev/null +++ b/accounts/abi/unpack.go @@ -0,0 +1,235 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package abi + +import ( + "encoding/binary" + "fmt" + "math/big" + "reflect" + + "github.com/ethereum/go-ethereum/common" +) + +// toGoSliceType parses the input and casts it to the proper slice defined by the ABI +// argument in T. +func toGoSlice(i int, t Argument, output []byte) (interface{}, error) { + index := i * 32 + // The slice must, at very least be large enough for the index+32 which is exactly the size required + // for the [offset in output, size of offset]. + if index+32 > len(output) { + return nil, fmt.Errorf("abi: cannot marshal in to go slice: insufficient size output %d require %d", len(output), index+32) + } + elem := t.Type.Elem + + // first we need to create a slice of the type + var refSlice reflect.Value + switch elem.T { + case IntTy, UintTy, BoolTy: + // create a new reference slice matching the element type + switch t.Type.Kind { + case reflect.Bool: + refSlice = reflect.ValueOf([]bool(nil)) + case reflect.Uint8: + refSlice = reflect.ValueOf([]uint8(nil)) + case reflect.Uint16: + refSlice = reflect.ValueOf([]uint16(nil)) + case reflect.Uint32: + refSlice = reflect.ValueOf([]uint32(nil)) + case reflect.Uint64: + refSlice = reflect.ValueOf([]uint64(nil)) + case reflect.Int8: + refSlice = reflect.ValueOf([]int8(nil)) + case reflect.Int16: + refSlice = reflect.ValueOf([]int16(nil)) + case reflect.Int32: + refSlice = reflect.ValueOf([]int32(nil)) + case reflect.Int64: + refSlice = reflect.ValueOf([]int64(nil)) + default: + refSlice = reflect.ValueOf([]*big.Int(nil)) + } + case AddressTy: // address must be of slice Address + refSlice = reflect.ValueOf([]common.Address(nil)) + case HashTy: // hash must be of slice hash + refSlice = reflect.ValueOf([]common.Hash(nil)) + case FixedBytesTy: + refSlice = reflect.ValueOf([][]byte(nil)) + default: // no other types are supported + return nil, fmt.Errorf("abi: unsupported slice type %v", elem.T) + } + + var slice []byte + var size int + var offset int + if t.Type.IsSlice { + // get the offset which determines the start of this array ... + offset = int(binary.BigEndian.Uint64(output[index+24 : index+32])) + if offset+32 > len(output) { + return nil, fmt.Errorf("abi: cannot marshal in to go slice: offset %d would go over slice boundary (len=%d)", len(output), offset+32) + } + + slice = output[offset:] + // ... starting with the size of the array in elements ... + size = int(binary.BigEndian.Uint64(slice[24:32])) + slice = slice[32:] + // ... and make sure that we've at the very least the amount of bytes + // available in the buffer. + if size*32 > len(slice) { + return nil, fmt.Errorf("abi: cannot marshal in to go slice: insufficient size output %d require %d", len(output), offset+32+size*32) + } + + // reslice to match the required size + slice = slice[:size*32] + } else if t.Type.IsArray { + //get the number of elements in the array + size = t.Type.SliceSize + + //check to make sure array size matches up + if index+32*size > len(output) { + return nil, fmt.Errorf("abi: cannot marshal in to go array: offset %d would go over slice boundary (len=%d)", len(output), index+32*size) + } + //slice is there for a fixed amount of times + slice = output[index : index+size*32] + } + + for i := 0; i < size; i++ { + var ( + inter interface{} // interface type + returnOutput = slice[i*32 : i*32+32] // the return output + err error + ) + // set inter to the correct type (cast) + switch elem.T { + case IntTy, UintTy: + inter = readInteger(t.Type.Kind, returnOutput) + case BoolTy: + inter, err = readBool(returnOutput) + if err != nil { + return nil, err + } + case AddressTy: + inter = common.BytesToAddress(returnOutput) + case HashTy: + inter = common.BytesToHash(returnOutput) + case FixedBytesTy: + inter = returnOutput + } + // append the item to our reflect slice + refSlice = reflect.Append(refSlice, reflect.ValueOf(inter)) + } + + // return the interface + return refSlice.Interface(), nil +} + +func readInteger(kind reflect.Kind, b []byte) interface{} { + switch kind { + case reflect.Uint8: + return uint8(b[len(b)-1]) + case reflect.Uint16: + return binary.BigEndian.Uint16(b[len(b)-2:]) + case reflect.Uint32: + return binary.BigEndian.Uint32(b[len(b)-4:]) + case reflect.Uint64: + return binary.BigEndian.Uint64(b[len(b)-8:]) + case reflect.Int8: + return int8(b[len(b)-1]) + case reflect.Int16: + return int16(binary.BigEndian.Uint16(b[len(b)-2:])) + case reflect.Int32: + return int32(binary.BigEndian.Uint32(b[len(b)-4:])) + case reflect.Int64: + return int64(binary.BigEndian.Uint64(b[len(b)-8:])) + default: + return new(big.Int).SetBytes(b) + } +} + +func readBool(word []byte) (bool, error) { + if len(word) != 32 { + return false, fmt.Errorf("abi: fatal error: incorrect word length") + } + + for i, b := range word { + if b != 0 && i != 31 { + return false, errBadBool + } + } + switch word[31] { + case 0: + return false, nil + case 1: + return true, nil + default: + return false, errBadBool + } + +} + +// toGoType parses the input and casts it to the proper type defined by the ABI +// argument in T. +func toGoType(i int, t Argument, output []byte) (interface{}, error) { + // we need to treat slices differently + if (t.Type.IsSlice || t.Type.IsArray) && t.Type.T != BytesTy && t.Type.T != StringTy && t.Type.T != FixedBytesTy && t.Type.T != FunctionTy { + return toGoSlice(i, t, output) + } + + index := i * 32 + if index+32 > len(output) { + return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), index+32) + } + + // Parse the given index output and check whether we need to read + // a different offset and length based on the type (i.e. string, bytes) + var returnOutput []byte + switch t.Type.T { + case StringTy, BytesTy: // variable arrays are written at the end of the return bytes + // parse offset from which we should start reading + offset := int(binary.BigEndian.Uint64(output[index+24 : index+32])) + if offset+32 > len(output) { + return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32) + } + // parse the size up until we should be reading + size := int(binary.BigEndian.Uint64(output[offset+24 : offset+32])) + if offset+32+size > len(output) { + return nil, fmt.Errorf("abi: cannot marshal in to go type: length insufficient %d require %d", len(output), offset+32+size) + } + + // get the bytes for this return value + returnOutput = output[offset+32 : offset+32+size] + default: + returnOutput = output[index : index+32] + } + + // convert the bytes to whatever is specified by the ABI. + switch t.Type.T { + case IntTy, UintTy: + return readInteger(t.Type.Kind, returnOutput), nil + case BoolTy: + return readBool(returnOutput) + case AddressTy: + return common.BytesToAddress(returnOutput), nil + case HashTy: + return common.BytesToHash(returnOutput), nil + case BytesTy, FixedBytesTy, FunctionTy: + return returnOutput, nil + case StringTy: + return string(returnOutput), nil + } + return nil, fmt.Errorf("abi: unknown type %v", t.Type.T) +} diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go new file mode 100644 index 0000000000..8e3afee4e6 --- /dev/null +++ b/accounts/abi/unpack_test.go @@ -0,0 +1,681 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package abi + +import ( + "bytes" + "fmt" + "math/big" + "reflect" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestSimpleMethodUnpack(t *testing.T) { + for i, test := range []struct { + def string // definition of the **output** ABI params + marshalledOutput []byte // evm return data + expectedOut interface{} // the expected output + outVar string // the output variable (e.g. uint32, *big.Int, etc) + err string // empty or error if expected + }{ + { + `[ { "type": "bool" } ]`, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), + bool(true), + "bool", + "", + }, + { + `[ { "type": "uint32" } ]`, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), + uint32(1), + "uint32", + "", + }, + { + `[ { "type": "uint32" } ]`, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), + nil, + "uint16", + "abi: cannot unmarshal uint32 in to uint16", + }, + { + `[ { "type": "uint17" } ]`, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), + nil, + "uint16", + "abi: cannot unmarshal *big.Int in to uint16", + }, + { + `[ { "type": "uint17" } ]`, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), + big.NewInt(1), + "*big.Int", + "", + }, + + { + `[ { "type": "int32" } ]`, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), + int32(1), + "int32", + "", + }, + { + `[ { "type": "int32" } ]`, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), + nil, + "int16", + "abi: cannot unmarshal int32 in to int16", + }, + { + `[ { "type": "int17" } ]`, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), + nil, + "int16", + "abi: cannot unmarshal *big.Int in to int16", + }, + { + `[ { "type": "int17" } ]`, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), + big.NewInt(1), + "*big.Int", + "", + }, + + { + `[ { "type": "address" } ]`, + common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000"), + common.Address{1}, + "address", + "", + }, + { + `[ { "type": "bytes32" } ]`, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + "bytes", + "", + }, + { + `[ { "type": "bytes32" } ]`, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + "hash", + "", + }, + { + `[ { "type": "bytes32" } ]`, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + "interface", + "", + }, + { + `[ { "type": "function" } ]`, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), + [24]byte{1}, + "function", + "", + }, + } { + abiDefinition := fmt.Sprintf(`[{ "name" : "method", "outputs": %s}]`, test.def) + abi, err := JSON(strings.NewReader(abiDefinition)) + if err != nil { + t.Errorf("%d failed. %v", i, err) + continue + } + + var outvar interface{} + switch test.outVar { + case "bool": + var v bool + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "uint8": + var v uint8 + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "uint16": + var v uint16 + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "uint32": + var v uint32 + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "uint64": + var v uint64 + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "int8": + var v int8 + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "int16": + var v int16 + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "int32": + var v int32 + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "int64": + var v int64 + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "*big.Int": + var v *big.Int + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "address": + var v common.Address + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "bytes": + var v []byte + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "hash": + var v common.Hash + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v.Bytes()[:] + case "function": + var v [24]byte + err = abi.Unpack(&v, "method", test.marshalledOutput) + outvar = v + case "interface": + err = abi.Unpack(&outvar, "method", test.marshalledOutput) + default: + t.Errorf("unsupported type '%v' please add it to the switch statement in this test", test.outVar) + continue + } + + if err != nil && len(test.err) == 0 { + t.Errorf("%d failed. Expected no err but got: %v", i, err) + continue + } + if err == nil && len(test.err) != 0 { + t.Errorf("%d failed. Expected err: %v but got none", i, test.err) + continue + } + if err != nil && len(test.err) != 0 && err.Error() != test.err { + t.Errorf("%d failed. Expected err: '%v' got err: '%v'", i, test.err, err) + continue + } + + if err == nil { + if !reflect.DeepEqual(test.expectedOut, outvar) { + t.Errorf("%d failed. Output error: expected %v, got %v", i, test.expectedOut, outvar) + } + } + } +} + +func TestUnpackSetInterfaceSlice(t *testing.T) { + var ( + var1 = new(uint8) + var2 = new(uint8) + ) + out := []interface{}{var1, var2} + abi, err := JSON(strings.NewReader(`[{"type":"function", "name":"ints", "outputs":[{"type":"uint8"}, {"type":"uint8"}]}]`)) + if err != nil { + t.Fatal(err) + } + marshalledReturn := append(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")...) + err = abi.Unpack(&out, "ints", marshalledReturn) + if err != nil { + t.Fatal(err) + } + if *var1 != 1 { + t.Error("expected var1 to be 1, got", *var1) + } + if *var2 != 2 { + t.Error("expected var2 to be 2, got", *var2) + } + + out = []interface{}{var1} + err = abi.Unpack(&out, "ints", marshalledReturn) + + expErr := "abi: cannot marshal in to slices of unequal size (require: 2, got: 1)" + if err == nil || err.Error() != expErr { + t.Error("expected err:", expErr, "Got:", err) + } +} + +func TestUnpackSetInterfaceArrayOutput(t *testing.T) { + var ( + var1 = new([1]uint32) + var2 = new([1]uint32) + ) + out := []interface{}{var1, var2} + abi, err := JSON(strings.NewReader(`[{"type":"function", "name":"ints", "outputs":[{"type":"uint32[1]"}, {"type":"uint32[1]"}]}]`)) + if err != nil { + t.Fatal(err) + } + marshalledReturn := append(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001"), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")...) + err = abi.Unpack(&out, "ints", marshalledReturn) + if err != nil { + t.Fatal(err) + } + + if *var1 != [1]uint32{1} { + t.Error("expected var1 to be [1], got", *var1) + } + if *var2 != [1]uint32{2} { + t.Error("expected var2 to be [2], got", *var2) + } +} + +func TestMultiReturnWithStruct(t *testing.T) { + const definition = `[ + { "name" : "multi", "constant" : false, "outputs": [ { "name": "Int", "type": "uint256" }, { "name": "String", "type": "string" } ] }]` + + abi, err := JSON(strings.NewReader(definition)) + if err != nil { + t.Fatal(err) + } + + // using buff to make the code readable + buff := new(bytes.Buffer) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000005")) + stringOut := "hello" + buff.Write(common.RightPadBytes([]byte(stringOut), 32)) + + var inter struct { + Int *big.Int + String string + } + err = abi.Unpack(&inter, "multi", buff.Bytes()) + if err != nil { + t.Error(err) + } + + if inter.Int == nil || inter.Int.Cmp(big.NewInt(1)) != 0 { + t.Error("expected Int to be 1 got", inter.Int) + } + + if inter.String != stringOut { + t.Error("expected String to be", stringOut, "got", inter.String) + } + + var reversed struct { + String string + Int *big.Int + } + + err = abi.Unpack(&reversed, "multi", buff.Bytes()) + if err != nil { + t.Error(err) + } + + if reversed.Int == nil || reversed.Int.Cmp(big.NewInt(1)) != 0 { + t.Error("expected Int to be 1 got", reversed.Int) + } + + if reversed.String != stringOut { + t.Error("expected String to be", stringOut, "got", reversed.String) + } +} + +func TestMultiReturnWithSlice(t *testing.T) { + const definition = `[ + { "name" : "multi", "constant" : false, "outputs": [ { "name": "Int", "type": "uint256" }, { "name": "String", "type": "string" } ] }]` + + abi, err := JSON(strings.NewReader(definition)) + if err != nil { + t.Fatal(err) + } + + // using buff to make the code readable + buff := new(bytes.Buffer) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000005")) + stringOut := "hello" + buff.Write(common.RightPadBytes([]byte(stringOut), 32)) + + var inter []interface{} + err = abi.Unpack(&inter, "multi", buff.Bytes()) + if err != nil { + t.Error(err) + } + + if len(inter) != 2 { + t.Fatal("expected 2 results got", len(inter)) + } + + if num, ok := inter[0].(*big.Int); !ok || num.Cmp(big.NewInt(1)) != 0 { + t.Error("expected index 0 to be 1 got", num) + } + + if str, ok := inter[1].(string); !ok || str != stringOut { + t.Error("expected index 1 to be", stringOut, "got", str) + } +} + +func TestMarshalArrays(t *testing.T) { + const definition = `[ + { "name" : "bytes32", "constant" : false, "outputs": [ { "type": "bytes32" } ] }, + { "name" : "bytes10", "constant" : false, "outputs": [ { "type": "bytes10" } ] } + ]` + + abi, err := JSON(strings.NewReader(definition)) + if err != nil { + t.Fatal(err) + } + + output := common.LeftPadBytes([]byte{1}, 32) + + var bytes10 [10]byte + err = abi.Unpack(&bytes10, "bytes32", output) + if err == nil || err.Error() != "abi: cannot unmarshal src (len=32) in to dst (len=10)" { + t.Error("expected error or bytes32 not be assignable to bytes10:", err) + } + + var bytes32 [32]byte + err = abi.Unpack(&bytes32, "bytes32", output) + if err != nil { + t.Error("didn't expect error:", err) + } + if !bytes.Equal(bytes32[:], output) { + t.Error("expected bytes32[31] to be 1 got", bytes32[31]) + } + + type ( + B10 [10]byte + B32 [32]byte + ) + + var b10 B10 + err = abi.Unpack(&b10, "bytes32", output) + if err == nil || err.Error() != "abi: cannot unmarshal src (len=32) in to dst (len=10)" { + t.Error("expected error or bytes32 not be assignable to bytes10:", err) + } + + var b32 B32 + err = abi.Unpack(&b32, "bytes32", output) + if err != nil { + t.Error("didn't expect error:", err) + } + if !bytes.Equal(b32[:], output) { + t.Error("expected bytes32[31] to be 1 got", bytes32[31]) + } + + output[10] = 1 + var shortAssignLong [32]byte + err = abi.Unpack(&shortAssignLong, "bytes10", output) + if err != nil { + t.Error("didn't expect error:", err) + } + if !bytes.Equal(output, shortAssignLong[:]) { + t.Errorf("expected %x to be %x", shortAssignLong, output) + } +} + +func TestUnmarshal(t *testing.T) { + const definition = `[ + { "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] }, + { "name" : "bool", "constant" : false, "outputs": [ { "type": "bool" } ] }, + { "name" : "bytes", "constant" : false, "outputs": [ { "type": "bytes" } ] }, + { "name" : "fixed", "constant" : false, "outputs": [ { "type": "bytes32" } ] }, + { "name" : "multi", "constant" : false, "outputs": [ { "type": "bytes" }, { "type": "bytes" } ] }, + { "name" : "intArraySingle", "constant" : false, "outputs": [ { "type": "uint256[3]" } ] }, + { "name" : "addressSliceSingle", "constant" : false, "outputs": [ { "type": "address[]" } ] }, + { "name" : "addressSliceDouble", "constant" : false, "outputs": [ { "name": "a", "type": "address[]" }, { "name": "b", "type": "address[]" } ] }, + { "name" : "mixedBytes", "constant" : true, "outputs": [ { "name": "a", "type": "bytes" }, { "name": "b", "type": "bytes32" } ] }]` + + abi, err := JSON(strings.NewReader(definition)) + if err != nil { + t.Fatal(err) + } + buff := new(bytes.Buffer) + + // marshal int + var Int *big.Int + err = abi.Unpack(&Int, "int", common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) + if err != nil { + t.Error(err) + } + + if Int == nil || Int.Cmp(big.NewInt(1)) != 0 { + t.Error("expected Int to be 1 got", Int) + } + + // marshal bool + var Bool bool + err = abi.Unpack(&Bool, "bool", common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) + if err != nil { + t.Error(err) + } + + if !Bool { + t.Error("expected Bool to be true") + } + + // marshal dynamic bytes max length 32 + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + bytesOut := common.RightPadBytes([]byte("hello"), 32) + buff.Write(bytesOut) + + var Bytes []byte + err = abi.Unpack(&Bytes, "bytes", buff.Bytes()) + if err != nil { + t.Error(err) + } + + if !bytes.Equal(Bytes, bytesOut) { + t.Errorf("expected %x got %x", bytesOut, Bytes) + } + + // marshall dynamic bytes max length 64 + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) + bytesOut = common.RightPadBytes([]byte("hello"), 64) + buff.Write(bytesOut) + + err = abi.Unpack(&Bytes, "bytes", buff.Bytes()) + if err != nil { + t.Error(err) + } + + if !bytes.Equal(Bytes, bytesOut) { + t.Errorf("expected %x got %x", bytesOut, Bytes) + } + + // marshall dynamic bytes max length 63 + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000003f")) + bytesOut = common.RightPadBytes([]byte("hello"), 63) + buff.Write(bytesOut) + + err = abi.Unpack(&Bytes, "bytes", buff.Bytes()) + if err != nil { + t.Error(err) + } + + if !bytes.Equal(Bytes, bytesOut) { + t.Errorf("expected %x got %x", bytesOut, Bytes) + } + + // marshal dynamic bytes output empty + err = abi.Unpack(&Bytes, "bytes", nil) + if err == nil { + t.Error("expected error") + } + + // marshal dynamic bytes length 5 + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000005")) + buff.Write(common.RightPadBytes([]byte("hello"), 32)) + + err = abi.Unpack(&Bytes, "bytes", buff.Bytes()) + if err != nil { + t.Error(err) + } + + if !bytes.Equal(Bytes, []byte("hello")) { + t.Errorf("expected %x got %x", bytesOut, Bytes) + } + + // marshal dynamic bytes length 5 + buff.Reset() + buff.Write(common.RightPadBytes([]byte("hello"), 32)) + + var hash common.Hash + err = abi.Unpack(&hash, "fixed", buff.Bytes()) + if err != nil { + t.Error(err) + } + + helloHash := common.BytesToHash(common.RightPadBytes([]byte("hello"), 32)) + if hash != helloHash { + t.Errorf("Expected %x to equal %x", hash, helloHash) + } + + // marshal error + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + err = abi.Unpack(&Bytes, "bytes", buff.Bytes()) + if err == nil { + t.Error("expected error") + } + + err = abi.Unpack(&Bytes, "multi", make([]byte, 64)) + if err == nil { + t.Error("expected error") + } + + // marshal mixed bytes + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) + fixed := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001") + buff.Write(fixed) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) + bytesOut = common.RightPadBytes([]byte("hello"), 32) + buff.Write(bytesOut) + + var out []interface{} + err = abi.Unpack(&out, "mixedBytes", buff.Bytes()) + if err != nil { + t.Fatal("didn't expect error:", err) + } + + if !bytes.Equal(bytesOut, out[0].([]byte)) { + t.Errorf("expected %x, got %x", bytesOut, out[0]) + } + + if !bytes.Equal(fixed, out[1].([]byte)) { + t.Errorf("expected %x, got %x", fixed, out[1]) + } + + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000003")) + // marshal int array + var intArray [3]*big.Int + err = abi.Unpack(&intArray, "intArraySingle", buff.Bytes()) + if err != nil { + t.Error(err) + } + var testAgainstIntArray [3]*big.Int + testAgainstIntArray[0] = big.NewInt(1) + testAgainstIntArray[1] = big.NewInt(2) + testAgainstIntArray[2] = big.NewInt(3) + + for i, Int := range intArray { + if Int.Cmp(testAgainstIntArray[i]) != 0 { + t.Errorf("expected %v, got %v", testAgainstIntArray[i], Int) + } + } + // marshal address slice + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000020")) // offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // size + buff.Write(common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000")) + + var outAddr []common.Address + err = abi.Unpack(&outAddr, "addressSliceSingle", buff.Bytes()) + if err != nil { + t.Fatal("didn't expect error:", err) + } + + if len(outAddr) != 1 { + t.Fatal("expected 1 item, got", len(outAddr)) + } + + if outAddr[0] != (common.Address{1}) { + t.Errorf("expected %x, got %x", common.Address{1}, outAddr[0]) + } + + // marshal multiple address slice + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) // offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000080")) // offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // size + buff.Write(common.Hex2Bytes("0000000000000000000000000100000000000000000000000000000000000000")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // size + buff.Write(common.Hex2Bytes("0000000000000000000000000200000000000000000000000000000000000000")) + buff.Write(common.Hex2Bytes("0000000000000000000000000300000000000000000000000000000000000000")) + + var outAddrStruct struct { + A []common.Address + B []common.Address + } + err = abi.Unpack(&outAddrStruct, "addressSliceDouble", buff.Bytes()) + if err != nil { + t.Fatal("didn't expect error:", err) + } + + if len(outAddrStruct.A) != 1 { + t.Fatal("expected 1 item, got", len(outAddrStruct.A)) + } + + if outAddrStruct.A[0] != (common.Address{1}) { + t.Errorf("expected %x, got %x", common.Address{1}, outAddrStruct.A[0]) + } + + if len(outAddrStruct.B) != 2 { + t.Fatal("expected 1 item, got", len(outAddrStruct.B)) + } + + if outAddrStruct.B[0] != (common.Address{2}) { + t.Errorf("expected %x, got %x", common.Address{2}, outAddrStruct.B[0]) + } + if outAddrStruct.B[1] != (common.Address{3}) { + t.Errorf("expected %x, got %x", common.Address{3}, outAddrStruct.B[1]) + } + + // marshal invalid address slice + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000100")) + + err = abi.Unpack(&outAddr, "addressSliceSingle", buff.Bytes()) + if err == nil { + t.Fatal("expected error:", err) + } +} From 4a741df757cf6b1d1b4c0e8c875c34b650f46167 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 23 Jun 2017 10:50:49 +0200 Subject: [PATCH 22/43] common/hexutil: wrap errors in json.UnmarshalTypeError This adds type and struct field context to error messages. Instead of "hex string of odd length" users will now see "json: cannot unmarshal hex string of odd length into Go struct field SendTxArgs.from of type common.Address". --- common/hexutil/hexutil.go | 25 +++++++++------ common/hexutil/json.go | 47 ++++++++++++++++++++------- common/hexutil/json_test.go | 64 ++++++++++++++++++------------------- common/types.go | 15 +++++++++ common/types_test.go | 10 +++--- 5 files changed, 102 insertions(+), 59 deletions(-) diff --git a/common/hexutil/hexutil.go b/common/hexutil/hexutil.go index 6b128ae360..582a67c22c 100644 --- a/common/hexutil/hexutil.go +++ b/common/hexutil/hexutil.go @@ -32,7 +32,6 @@ package hexutil import ( "encoding/hex" - "errors" "fmt" "math/big" "strconv" @@ -41,17 +40,23 @@ import ( const uintBits = 32 << (uint64(^uint(0)) >> 63) var ( - ErrEmptyString = errors.New("empty hex string") - ErrMissingPrefix = errors.New("missing 0x prefix for hex data") - ErrSyntax = errors.New("invalid hex") - ErrEmptyNumber = errors.New("hex number has no digits after 0x") - ErrLeadingZero = errors.New("hex number has leading zero digits after 0x") - ErrOddLength = errors.New("hex string has odd length") - ErrUint64Range = errors.New("hex number does not fit into 64 bits") - ErrUintRange = fmt.Errorf("hex number does not fit into %d bits", uintBits) - ErrBig256Range = errors.New("hex number does not fit into 256 bits") + ErrEmptyString = &decError{"empty hex string"} + ErrSyntax = &decError{"invalid hex string"} + ErrMissingPrefix = &decError{"hex string without 0x prefix"} + ErrOddLength = &decError{"hex string of odd length"} + ErrEmptyNumber = &decError{"hex string \"0x\""} + ErrLeadingZero = &decError{"hex number with leading zero digits"} + ErrUint64Range = &decError{"hex number > 64 bits"} + ErrUintRange = &decError{fmt.Sprintf("hex number > %d bits", uintBits)} + ErrBig256Range = &decError{"hex number > 256 bits"} ) +type decError struct{ msg string } + +func (err decError) Error() string { + return string(err.msg) +} + // Decode decodes a hex string with 0x prefix. func Decode(input string) ([]byte, error) { if len(input) == 0 { diff --git a/common/hexutil/json.go b/common/hexutil/json.go index 1bc1d014c7..943288fad3 100644 --- a/common/hexutil/json.go +++ b/common/hexutil/json.go @@ -18,15 +18,19 @@ package hexutil import ( "encoding/hex" - "errors" + "encoding/json" "fmt" "math/big" + "reflect" "strconv" ) var ( - textZero = []byte(`0x0`) - errNonString = errors.New("cannot unmarshal non-string as hex data") + textZero = []byte(`0x0`) + bytesT = reflect.TypeOf(Bytes(nil)) + bigT = reflect.TypeOf((*Big)(nil)) + uintT = reflect.TypeOf(Uint(0)) + uint64T = reflect.TypeOf(Uint64(0)) ) // Bytes marshals/unmarshals as a JSON string with 0x prefix. @@ -44,9 +48,9 @@ func (b Bytes) MarshalText() ([]byte, error) { // UnmarshalJSON implements json.Unmarshaler. func (b *Bytes) UnmarshalJSON(input []byte) error { if !isString(input) { - return errNonString + return errNonString(bytesT) } - return b.UnmarshalText(input[1 : len(input)-1]) + return wrapTypeError(b.UnmarshalText(input[1:len(input)-1]), bytesT) } // UnmarshalText implements encoding.TextUnmarshaler. @@ -69,6 +73,16 @@ func (b Bytes) String() string { return Encode(b) } +// UnmarshalFixedJSON decodes the input as a string with 0x prefix. The length of out +// determines the required input length. This function is commonly used to implement the +// UnmarshalJSON method for fixed-size types. +func UnmarshalFixedJSON(typ reflect.Type, input, out []byte) error { + if !isString(input) { + return errNonString(typ) + } + return wrapTypeError(UnmarshalFixedText(typ.String(), input[1:len(input)-1], out), typ) +} + // UnmarshalFixedText decodes the input as a string with 0x prefix. The length of out // determines the required input length. This function is commonly used to implement the // UnmarshalText method for fixed-size types. @@ -127,9 +141,9 @@ func (b Big) MarshalText() ([]byte, error) { // UnmarshalJSON implements json.Unmarshaler. func (b *Big) UnmarshalJSON(input []byte) error { if !isString(input) { - return errNonString + return errNonString(bigT) } - return b.UnmarshalText(input[1 : len(input)-1]) + return wrapTypeError(b.UnmarshalText(input[1:len(input)-1]), bigT) } // UnmarshalText implements encoding.TextUnmarshaler @@ -189,9 +203,9 @@ func (b Uint64) MarshalText() ([]byte, error) { // UnmarshalJSON implements json.Unmarshaler. func (b *Uint64) UnmarshalJSON(input []byte) error { if !isString(input) { - return errNonString + return errNonString(uint64T) } - return b.UnmarshalText(input[1 : len(input)-1]) + return wrapTypeError(b.UnmarshalText(input[1:len(input)-1]), uint64T) } // UnmarshalText implements encoding.TextUnmarshaler @@ -233,9 +247,9 @@ func (b Uint) MarshalText() ([]byte, error) { // UnmarshalJSON implements json.Unmarshaler. func (b *Uint) UnmarshalJSON(input []byte) error { if !isString(input) { - return errNonString + return errNonString(uintT) } - return b.UnmarshalText(input[1 : len(input)-1]) + return wrapTypeError(b.UnmarshalText(input[1:len(input)-1]), uintT) } // UnmarshalText implements encoding.TextUnmarshaler. @@ -295,3 +309,14 @@ func checkNumberText(input []byte) (raw []byte, err error) { } return input, nil } + +func wrapTypeError(err error, typ reflect.Type) error { + if _, ok := err.(*decError); ok { + return &json.UnmarshalTypeError{Value: err.Error(), Type: typ} + } + return err +} + +func errNonString(typ reflect.Type) error { + return &json.UnmarshalTypeError{Value: "non-string", Type: typ} +} diff --git a/common/hexutil/json_test.go b/common/hexutil/json_test.go index e4e8274915..8a6b8643a1 100644 --- a/common/hexutil/json_test.go +++ b/common/hexutil/json_test.go @@ -62,12 +62,12 @@ var errJSONEOF = errors.New("unexpected end of JSON input") var unmarshalBytesTests = []unmarshalTest{ // invalid encoding {input: "", wantErr: errJSONEOF}, - {input: "null", wantErr: errNonString}, - {input: "10", wantErr: errNonString}, - {input: `"0"`, wantErr: ErrMissingPrefix}, - {input: `"0x0"`, wantErr: ErrOddLength}, - {input: `"0xxx"`, wantErr: ErrSyntax}, - {input: `"0x01zz01"`, wantErr: ErrSyntax}, + {input: "null", wantErr: errNonString(bytesT)}, + {input: "10", wantErr: errNonString(bytesT)}, + {input: `"0"`, wantErr: wrapTypeError(ErrMissingPrefix, bytesT)}, + {input: `"0x0"`, wantErr: wrapTypeError(ErrOddLength, bytesT)}, + {input: `"0xxx"`, wantErr: wrapTypeError(ErrSyntax, bytesT)}, + {input: `"0x01zz01"`, wantErr: wrapTypeError(ErrSyntax, bytesT)}, // valid encoding {input: `""`, want: referenceBytes("")}, @@ -127,16 +127,16 @@ func TestMarshalBytes(t *testing.T) { var unmarshalBigTests = []unmarshalTest{ // invalid encoding {input: "", wantErr: errJSONEOF}, - {input: "null", wantErr: errNonString}, - {input: "10", wantErr: errNonString}, - {input: `"0"`, wantErr: ErrMissingPrefix}, - {input: `"0x"`, wantErr: ErrEmptyNumber}, - {input: `"0x01"`, wantErr: ErrLeadingZero}, - {input: `"0xx"`, wantErr: ErrSyntax}, - {input: `"0x1zz01"`, wantErr: ErrSyntax}, + {input: "null", wantErr: errNonString(bigT)}, + {input: "10", wantErr: errNonString(bigT)}, + {input: `"0"`, wantErr: wrapTypeError(ErrMissingPrefix, bigT)}, + {input: `"0x"`, wantErr: wrapTypeError(ErrEmptyNumber, bigT)}, + {input: `"0x01"`, wantErr: wrapTypeError(ErrLeadingZero, bigT)}, + {input: `"0xx"`, wantErr: wrapTypeError(ErrSyntax, bigT)}, + {input: `"0x1zz01"`, wantErr: wrapTypeError(ErrSyntax, bigT)}, { input: `"0x10000000000000000000000000000000000000000000000000000000000000000"`, - wantErr: ErrBig256Range, + wantErr: wrapTypeError(ErrBig256Range, bigT), }, // valid encoding @@ -208,14 +208,14 @@ func TestMarshalBig(t *testing.T) { var unmarshalUint64Tests = []unmarshalTest{ // invalid encoding {input: "", wantErr: errJSONEOF}, - {input: "null", wantErr: errNonString}, - {input: "10", wantErr: errNonString}, - {input: `"0"`, wantErr: ErrMissingPrefix}, - {input: `"0x"`, wantErr: ErrEmptyNumber}, - {input: `"0x01"`, wantErr: ErrLeadingZero}, - {input: `"0xfffffffffffffffff"`, wantErr: ErrUint64Range}, - {input: `"0xx"`, wantErr: ErrSyntax}, - {input: `"0x1zz01"`, wantErr: ErrSyntax}, + {input: "null", wantErr: errNonString(uint64T)}, + {input: "10", wantErr: errNonString(uint64T)}, + {input: `"0"`, wantErr: wrapTypeError(ErrMissingPrefix, uint64T)}, + {input: `"0x"`, wantErr: wrapTypeError(ErrEmptyNumber, uint64T)}, + {input: `"0x01"`, wantErr: wrapTypeError(ErrLeadingZero, uint64T)}, + {input: `"0xfffffffffffffffff"`, wantErr: wrapTypeError(ErrUint64Range, uint64T)}, + {input: `"0xx"`, wantErr: wrapTypeError(ErrSyntax, uint64T)}, + {input: `"0x1zz01"`, wantErr: wrapTypeError(ErrSyntax, uint64T)}, // valid encoding {input: `""`, want: uint64(0)}, @@ -298,15 +298,15 @@ var ( var unmarshalUintTests = []unmarshalTest{ // invalid encoding {input: "", wantErr: errJSONEOF}, - {input: "null", wantErr: errNonString}, - {input: "10", wantErr: errNonString}, - {input: `"0"`, wantErr: ErrMissingPrefix}, - {input: `"0x"`, wantErr: ErrEmptyNumber}, - {input: `"0x01"`, wantErr: ErrLeadingZero}, - {input: `"0x100000000"`, want: uint(maxUint33bits), wantErr32bit: ErrUintRange}, - {input: `"0xfffffffffffffffff"`, wantErr: ErrUintRange}, - {input: `"0xx"`, wantErr: ErrSyntax}, - {input: `"0x1zz01"`, wantErr: ErrSyntax}, + {input: "null", wantErr: errNonString(uintT)}, + {input: "10", wantErr: errNonString(uintT)}, + {input: `"0"`, wantErr: wrapTypeError(ErrMissingPrefix, uintT)}, + {input: `"0x"`, wantErr: wrapTypeError(ErrEmptyNumber, uintT)}, + {input: `"0x01"`, wantErr: wrapTypeError(ErrLeadingZero, uintT)}, + {input: `"0x100000000"`, want: uint(maxUint33bits), wantErr32bit: wrapTypeError(ErrUintRange, uintT)}, + {input: `"0xfffffffffffffffff"`, wantErr: wrapTypeError(ErrUintRange, uintT)}, + {input: `"0xx"`, wantErr: wrapTypeError(ErrSyntax, uintT)}, + {input: `"0x1zz01"`, wantErr: wrapTypeError(ErrSyntax, uintT)}, // valid encoding {input: `""`, want: uint(0)}, @@ -317,7 +317,7 @@ var unmarshalUintTests = []unmarshalTest{ {input: `"0x1122aaff"`, want: uint(0x1122aaff)}, {input: `"0xbbb"`, want: uint(0xbbb)}, {input: `"0xffffffff"`, want: uint(0xffffffff)}, - {input: `"0xffffffffffffffff"`, want: uint(maxUint64bits), wantErr32bit: ErrUintRange}, + {input: `"0xffffffffffffffff"`, want: uint(maxUint64bits), wantErr32bit: wrapTypeError(ErrUintRange, uintT)}, } func TestUnmarshalUint(t *testing.T) { diff --git a/common/types.go b/common/types.go index 05288bf463..8037266349 100644 --- a/common/types.go +++ b/common/types.go @@ -31,6 +31,11 @@ const ( AddressLength = 20 ) +var ( + hashT = reflect.TypeOf(Hash{}) + addressT = reflect.TypeOf(Address{}) +) + // Hash represents the 32 byte Keccak256 hash of arbitrary data. type Hash [HashLength]byte @@ -72,6 +77,11 @@ func (h *Hash) UnmarshalText(input []byte) error { return hexutil.UnmarshalFixedText("Hash", input, h[:]) } +// UnmarshalJSON parses a hash in hex syntax. +func (h *Hash) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(hashT, input, h[:]) +} + // MarshalText returns the hex representation of h. func (h Hash) MarshalText() ([]byte, error) { return hexutil.Bytes(h[:]).MarshalText() @@ -194,6 +204,11 @@ func (a *Address) UnmarshalText(input []byte) error { return hexutil.UnmarshalFixedText("Address", input, a[:]) } +// UnmarshalJSON parses a hash in hex syntax. +func (a *Address) UnmarshalJSON(input []byte) error { + return hexutil.UnmarshalFixedJSON(addressT, input, a[:]) +} + // UnprefixedHash allows marshaling an Address without 0x prefix. type UnprefixedAddress Address diff --git a/common/types_test.go b/common/types_test.go index 9f9d8b767c..154c330632 100644 --- a/common/types_test.go +++ b/common/types_test.go @@ -21,8 +21,6 @@ import ( "math/big" "strings" "testing" - - "github.com/ethereum/go-ethereum/common/hexutil" ) func TestBytesConversion(t *testing.T) { @@ -43,10 +41,10 @@ func TestHashJsonValidation(t *testing.T) { Size int Error string }{ - {"", 62, hexutil.ErrMissingPrefix.Error()}, - {"0x", 66, "hex string has length 66, want 64 for Hash"}, - {"0x", 63, hexutil.ErrOddLength.Error()}, - {"0x", 0, "hex string has length 0, want 64 for Hash"}, + {"", 62, "json: cannot unmarshal hex string without 0x prefix into Go value of type common.Hash"}, + {"0x", 66, "hex string has length 66, want 64 for common.Hash"}, + {"0x", 63, "json: cannot unmarshal hex string of odd length into Go value of type common.Hash"}, + {"0x", 0, "hex string has length 0, want 64 for common.Hash"}, {"0x", 64, ""}, {"0X", 64, ""}, } From 9e5f03b6c487175cc5aa1224e5e12fd573f483a7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 27 Jun 2017 15:57:06 +0200 Subject: [PATCH 23/43] core/state: access trie through Database interface, track errors (#14589) With this commit, core/state's access to the underlying key/value database is mediated through an interface. Database errors are tracked in StateDB and returned by CommitTo or the new Error method. Motivation for this change: We can remove the light client's duplicated copy of core/state. The light client now supports node iteration, so tracing and storage enumeration can work with the light client (not implemented in this commit). --- accounts/abi/bind/backends/simulated.go | 4 +- cmd/evm/runner.go | 6 +- cmd/geth/chaincmd.go | 2 +- core/block_validator.go | 12 +- core/blockchain.go | 38 ++- core/blockchain_test.go | 10 +- core/chain_makers.go | 4 +- core/genesis.go | 2 +- core/state/database.go | 154 ++++++++++++ core/state/dump.go | 2 +- core/state/iterator.go | 13 +- core/state/iterator_test.go | 8 +- core/state/managed_state_test.go | 2 +- core/state/state_object.go | 56 +++-- core/state/state_test.go | 39 ++- core/state/statedb.go | 139 +++-------- core/state/statedb_test.go | 40 +-- core/state/sync_test.go | 75 +++--- core/tx_pool_test.go | 24 +- core/vm/runtime/runtime.go | 4 +- core/vm/runtime/runtime_test.go | 2 +- eth/api.go | 2 +- eth/api_backend.go | 35 +-- eth/api_test.go | 2 +- eth/bind.go | 6 +- eth/downloader/downloader_test.go | 2 +- eth/handler_test.go | 2 +- internal/ethapi/api.go | 33 +-- internal/ethapi/backend.go | 13 +- les/api_backend.go | 20 +- les/odr_test.go | 53 ++-- les/request_test.go | 2 +- light/lightchain.go | 5 - light/odr.go | 8 +- light/odr_test.go | 164 ++++++------ light/odr_util.go | 19 -- light/state.go | 316 ------------------------ light/state_object.go | 275 --------------------- light/state_test.go | 248 ------------------- light/trie.go | 253 ++++++++++++++----- light/trie_test.go | 83 +++++++ light/txpool.go | 32 +-- light/vm_env.go | 194 --------------- miner/worker.go | 2 +- tests/block_test_util.go | 4 +- tests/state_test_util.go | 48 +--- tests/util.go | 7 +- trie/proof.go | 5 +- trie/secure_trie.go | 5 + 49 files changed, 810 insertions(+), 1664 deletions(-) create mode 100644 core/state/database.go delete mode 100644 light/state.go delete mode 100644 light/state_object.go delete mode 100644 light/state_test.go create mode 100644 light/trie_test.go delete mode 100644 light/vm_env.go diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 159fca136e..7ac8b58204 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -90,7 +90,7 @@ func (b *SimulatedBackend) Rollback() { func (b *SimulatedBackend) rollback() { blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), b.database, 1, func(int, *core.BlockGen) {}) b.pendingBlock = blocks[0] - b.pendingState, _ = state.New(b.pendingBlock.Root(), b.database) + b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database)) } // CodeAt returns the code associated with a certain account in the blockchain. @@ -279,7 +279,7 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa block.AddTx(tx) }) b.pendingBlock = blocks[0] - b.pendingState, _ = state.New(b.pendingBlock.Root(), b.database) + b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database)) return nil } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 2ce0920f69..3f95a0c93a 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -98,8 +98,8 @@ func runCmd(ctx *cli.Context) error { _, statedb = gen.ToBlock() chainConfig = gen.Config } else { - var db, _ = ethdb.NewMemDatabase() - statedb, _ = state.New(common.Hash{}, db) + db, _ := ethdb.NewMemDatabase() + statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) } if ctx.GlobalString(SenderFlag.Name) != "" { sender = common.HexToAddress(ctx.GlobalString(SenderFlag.Name)) @@ -188,7 +188,7 @@ func runCmd(ctx *cli.Context) error { execTime := time.Since(tstart) if ctx.GlobalBool(DumpFlag.Name) { - statedb.Commit(true) + statedb.IntermediateRoot(true) fmt.Println(string(statedb.Dump())) } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index ab0e92f213..12bc1d7c66 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -312,7 +312,7 @@ func dump(ctx *cli.Context) error { fmt.Println("{}") utils.Fatalf("block not found") } else { - state, err := state.New(block.Root(), chainDb) + state, err := state.New(block.Root(), state.NewDatabase(chainDb)) if err != nil { utils.Fatalf("could not create new state: %v", err) } diff --git a/core/block_validator.go b/core/block_validator.go index 4f85df0070..e9cfd04827 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -52,16 +52,10 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin // validated at this point. func (v *BlockValidator) ValidateBody(block *types.Block) error { // Check whether the block's known, and if not, that it's linkable - if v.bc.HasBlock(block.Hash()) { - if _, err := state.New(block.Root(), v.bc.chainDb); err == nil { - return ErrKnownBlock - } + if v.bc.HasBlockAndState(block.Hash()) { + return ErrKnownBlock } - parent := v.bc.GetBlock(block.ParentHash(), block.NumberU64()-1) - if parent == nil { - return consensus.ErrUnknownAncestor - } - if _, err := state.New(parent.Root(), v.bc.chainDb); err != nil { + if !v.bc.HasBlockAndState(block.ParentHash()) { return consensus.ErrUnknownAncestor } // Header validity is known at this point, check the uncles and transactions diff --git a/core/blockchain.go b/core/blockchain.go index 073b91bab4..aab2e72f3a 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -92,7 +92,7 @@ type BlockChain struct { currentBlock *types.Block // Current head of the block chain currentFastBlock *types.Block // Current head of the fast-sync chain (may be above the block chain!) - stateCache *state.StateDB // State database to reuse between imports (contains state cache) + stateCache state.Database // State database to reuse between imports (contains state cache) bodyCache *lru.Cache // Cache for the most recent block bodies bodyRLPCache *lru.Cache // Cache for the most recent block bodies in RLP encoded format blockCache *lru.Cache // Cache for the most recent entire blocks @@ -125,6 +125,7 @@ func NewBlockChain(chainDb ethdb.Database, config *params.ChainConfig, engine co bc := &BlockChain{ config: config, chainDb: chainDb, + stateCache: state.NewDatabase(chainDb), eventMux: mux, quit: make(chan struct{}), bodyCache: bodyCache, @@ -190,7 +191,7 @@ func (bc *BlockChain) loadLastState() error { return bc.Reset() } // Make sure the state associated with the block is available - if _, err := state.New(currentBlock.Root(), bc.chainDb); err != nil { + if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil { // Dangling block without a state associated, init from scratch log.Warn("Head state missing, resetting chain", "number", currentBlock.Number(), "hash", currentBlock.Hash()) return bc.Reset() @@ -214,12 +215,6 @@ func (bc *BlockChain) loadLastState() error { bc.currentFastBlock = block } } - // Initialize a statedb cache to ensure singleton account bloom filter generation - statedb, err := state.New(bc.currentBlock.Root(), bc.chainDb) - if err != nil { - return err - } - bc.stateCache = statedb // Issue a status log for the user headerTd := bc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()) @@ -261,7 +256,7 @@ func (bc *BlockChain) SetHead(head uint64) error { bc.currentBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) } if bc.currentBlock != nil { - if _, err := state.New(bc.currentBlock.Root(), bc.chainDb); err != nil { + if _, err := state.New(bc.currentBlock.Root(), bc.stateCache); err != nil { // Rewound state missing, rolled back to before pivot, reset to genesis bc.currentBlock = nil } @@ -384,7 +379,7 @@ func (bc *BlockChain) State() (*state.StateDB, error) { // StateAt returns a new mutable state based on a particular point in time. func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { - return bc.stateCache.New(root) + return state.New(root, bc.stateCache) } // Reset purges the entire blockchain, restoring it to its genesis state. @@ -531,7 +526,7 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash) bool { return false } // Ensure the associated state is also present - _, err := state.New(block.Root(), bc.chainDb) + _, err := bc.stateCache.OpenTrie(block.Root()) return err == nil } @@ -959,31 +954,30 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { } // Create a new statedb using the parent block and report an // error if it fails. - switch { - case i == 0: - err = bc.stateCache.Reset(bc.GetBlock(block.ParentHash(), block.NumberU64()-1).Root()) - default: - err = bc.stateCache.Reset(chain[i-1].Root()) + var parent *types.Block + if i == 0 { + parent = bc.GetBlock(block.ParentHash(), block.NumberU64()-1) + } else { + parent = chain[i-1] } + state, err := state.New(parent.Root(), bc.stateCache) if err != nil { - bc.reportBlock(block, nil, err) return i, err } // Process block using the parent state as reference point. - receipts, logs, usedGas, err := bc.processor.Process(block, bc.stateCache, bc.vmConfig) + receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig) if err != nil { bc.reportBlock(block, receipts, err) return i, err } // Validate the state using the default validator - err = bc.Validator().ValidateState(block, bc.GetBlock(block.ParentHash(), block.NumberU64()-1), bc.stateCache, receipts, usedGas) + err = bc.Validator().ValidateState(block, parent, state, receipts, usedGas) if err != nil { bc.reportBlock(block, receipts, err) return i, err } // Write state changes to database - _, err = bc.stateCache.Commit(bc.config.IsEIP158(block.Number())) - if err != nil { + if _, err = state.CommitTo(bc.chainDb, bc.config.IsEIP158(block.Number())); err != nil { return i, err } @@ -1021,7 +1015,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { return i, err } // Write hash preimages - if err := WritePreimages(bc.chainDb, block.NumberU64(), bc.stateCache.Preimages()); err != nil { + if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil { return i, err } case SideStatTy: diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 7505208e18..371522ab77 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -131,7 +131,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { } return err } - statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.chainDb) + statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.stateCache) if err != nil { return err } @@ -148,7 +148,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { blockchain.mu.Lock() WriteTd(blockchain.chainDb, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash()))) WriteBlock(blockchain.chainDb, block) - statedb.Commit(false) + statedb.CommitTo(blockchain.chainDb, false) blockchain.mu.Unlock() } return nil @@ -1131,7 +1131,7 @@ func TestEIP161AccountRemoval(t *testing.T) { if _, err := blockchain.InsertChain(types.Blocks{blocks[0]}); err != nil { t.Fatal(err) } - if !blockchain.stateCache.Exist(theAddr) { + if st, _ := blockchain.State(); !st.Exist(theAddr) { t.Error("expected account to exist") } @@ -1139,7 +1139,7 @@ func TestEIP161AccountRemoval(t *testing.T) { if _, err := blockchain.InsertChain(types.Blocks{blocks[1]}); err != nil { t.Fatal(err) } - if blockchain.stateCache.Exist(theAddr) { + if st, _ := blockchain.State(); st.Exist(theAddr) { t.Error("account should not exist") } @@ -1147,7 +1147,7 @@ func TestEIP161AccountRemoval(t *testing.T) { if _, err := blockchain.InsertChain(types.Blocks{blocks[2]}); err != nil { t.Fatal(err) } - if blockchain.stateCache.Exist(theAddr) { + if st, _ := blockchain.State(); st.Exist(theAddr) { t.Error("account should not exist") } } diff --git a/core/chain_makers.go b/core/chain_makers.go index cc14f8fb82..38a69d42a9 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -181,7 +181,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, db ethdb.Dat gen(i, b) } ethash.AccumulateRewards(statedb, h, b.uncles) - root, err := statedb.Commit(config.IsEIP158(h.Number)) + root, err := statedb.CommitTo(db, config.IsEIP158(h.Number)) if err != nil { panic(fmt.Sprintf("state write error: %v", err)) } @@ -189,7 +189,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, db ethdb.Dat return types.NewBlock(h, b.txs, b.uncles, b.receipts), b.receipts } for i := 0; i < n; i++ { - statedb, err := state.New(parent.Root(), db) + statedb, err := state.New(parent.Root(), state.NewDatabase(db)) if err != nil { panic(err) } diff --git a/core/genesis.go b/core/genesis.go index 947a53c700..5815d5901b 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -176,7 +176,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { // ToBlock creates the block and state of a genesis specification. func (g *Genesis) ToBlock() (*types.Block, *state.StateDB) { db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) for addr, account := range g.Alloc { statedb.AddBalance(addr, account.Balance) statedb.SetCode(addr, account.Code) diff --git a/core/state/database.go b/core/state/database.go new file mode 100644 index 0000000000..946625e76e --- /dev/null +++ b/core/state/database.go @@ -0,0 +1,154 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" + lru "github.com/hashicorp/golang-lru" +) + +// Trie cache generation limit after which to evic trie nodes from memory. +var MaxTrieCacheGen = uint16(120) + +const ( + // Number of past tries to keep. This value is chosen such that + // reasonable chain reorg depths will hit an existing trie. + maxPastTries = 12 + + // Number of codehash->size associations to keep. + codeSizeCacheSize = 100000 +) + +// Database wraps access to tries and contract code. +type Database interface { + // Accessing tries: + // OpenTrie opens the main account trie. + // OpenStorageTrie opens the storage trie of an account. + OpenTrie(root common.Hash) (Trie, error) + OpenStorageTrie(addrHash, root common.Hash) (Trie, error) + // Accessing contract code: + ContractCode(addrHash, codeHash common.Hash) ([]byte, error) + ContractCodeSize(addrHash, codeHash common.Hash) (int, error) + // CopyTrie returns an independent copy of the given trie. + CopyTrie(Trie) Trie +} + +// Trie is a Ethereum Merkle Trie. +type Trie interface { + TryGet(key []byte) ([]byte, error) + TryUpdate(key, value []byte) error + TryDelete(key []byte) error + CommitTo(trie.DatabaseWriter) (common.Hash, error) + Hash() common.Hash + NodeIterator(startKey []byte) trie.NodeIterator + GetKey([]byte) []byte // TODO(fjl): remove this when SecureTrie is removed +} + +// NewDatabase creates a backing store for state. The returned database is safe for +// concurrent use and retains cached trie nodes in memory. +func NewDatabase(db ethdb.Database) Database { + csc, _ := lru.New(codeSizeCacheSize) + return &cachingDB{db: db, codeSizeCache: csc} +} + +type cachingDB struct { + db ethdb.Database + mu sync.Mutex + pastTries []*trie.SecureTrie + codeSizeCache *lru.Cache +} + +func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { + db.mu.Lock() + defer db.mu.Unlock() + + for i := len(db.pastTries) - 1; i >= 0; i-- { + if db.pastTries[i].Hash() == root { + return cachedTrie{db.pastTries[i].Copy(), db}, nil + } + } + tr, err := trie.NewSecure(root, db.db, MaxTrieCacheGen) + if err != nil { + return nil, err + } + return cachedTrie{tr, db}, nil +} + +func (db *cachingDB) pushTrie(t *trie.SecureTrie) { + db.mu.Lock() + defer db.mu.Unlock() + + if len(db.pastTries) >= maxPastTries { + copy(db.pastTries, db.pastTries[1:]) + db.pastTries[len(db.pastTries)-1] = t + } else { + db.pastTries = append(db.pastTries, t) + } +} + +func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { + return trie.NewSecure(root, db.db, 0) +} + +func (db *cachingDB) CopyTrie(t Trie) Trie { + switch t := t.(type) { + case cachedTrie: + return cachedTrie{t.SecureTrie.Copy(), db} + case *trie.SecureTrie: + return t.Copy() + default: + panic(fmt.Errorf("unknown trie type %T", t)) + } +} + +func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { + code, err := db.db.Get(codeHash[:]) + if err == nil { + db.codeSizeCache.Add(codeHash, len(code)) + } + return code, err +} + +func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, error) { + if cached, ok := db.codeSizeCache.Get(codeHash); ok { + return cached.(int), nil + } + code, err := db.ContractCode(addrHash, codeHash) + if err == nil { + db.codeSizeCache.Add(codeHash, len(code)) + } + return len(code), err +} + +// cachedTrie inserts its trie into a cachingDB on commit. +type cachedTrie struct { + *trie.SecureTrie + db *cachingDB +} + +func (m cachedTrie) CommitTo(dbw trie.DatabaseWriter) (common.Hash, error) { + root, err := m.SecureTrie.CommitTo(dbw) + if err == nil { + m.db.pushTrie(m.SecureTrie) + } + return root, err +} diff --git a/core/state/dump.go b/core/state/dump.go index ffa1a72832..46e612850a 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -41,7 +41,7 @@ type Dump struct { func (self *StateDB) RawDump() Dump { dump := Dump{ - Root: common.Bytes2Hex(self.trie.Root()), + Root: fmt.Sprintf("%x", self.trie.Hash()), Accounts: make(map[string]DumpAccount), } diff --git a/core/state/iterator.go b/core/state/iterator.go index a8a2722ae3..6a5c73d3d1 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -19,7 +19,6 @@ package state import ( "bytes" "fmt" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" @@ -105,16 +104,11 @@ func (it *NodeIterator) step() error { return nil } // Otherwise we've reached an account node, initiate data iteration - var account struct { - Nonce uint64 - Balance *big.Int - Root common.Hash - CodeHash []byte - } + var account Account if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil { return err } - dataTrie, err := trie.New(account.Root, it.state.db) + dataTrie, err := it.state.db.OpenStorageTrie(common.BytesToHash(it.stateIt.LeafKey()), account.Root) if err != nil { return err } @@ -124,7 +118,8 @@ func (it *NodeIterator) step() error { } if !bytes.Equal(account.CodeHash, emptyCodeHash) { it.codeHash = common.BytesToHash(account.CodeHash) - it.code, err = it.state.db.Get(account.CodeHash) + addrHash := common.BytesToHash(it.stateIt.LeafKey()) + it.code, err = it.state.db.ContractCode(addrHash, common.BytesToHash(account.CodeHash)) if err != nil { return fmt.Errorf("code %x: %v", account.CodeHash, err) } diff --git a/core/state/iterator_test.go b/core/state/iterator_test.go index aa9c5b7288..ff66ba7a94 100644 --- a/core/state/iterator_test.go +++ b/core/state/iterator_test.go @@ -21,13 +21,12 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethdb" ) // Tests that the node iterator indeed walks over the entire database contents. func TestNodeIteratorCoverage(t *testing.T) { // Create some arbitrary test state to iterate - db, root, _ := makeTestState() + db, mem, root, _ := makeTestState() state, err := New(root, db) if err != nil { @@ -40,13 +39,14 @@ func TestNodeIteratorCoverage(t *testing.T) { hashes[it.Hash] = struct{}{} } } + // Cross check the hashes and the database itself for hash := range hashes { - if _, err := db.Get(hash.Bytes()); err != nil { + if _, err := mem.Get(hash.Bytes()); err != nil { t.Errorf("failed to retrieve reported node %x: %v", hash, err) } } - for _, key := range db.(*ethdb.MemDatabase).Keys() { + for _, key := range mem.Keys() { if bytes.HasPrefix(key, []byte("secure-key-")) { continue } diff --git a/core/state/managed_state_test.go b/core/state/managed_state_test.go index ea5737a089..1cfdd3a890 100644 --- a/core/state/managed_state_test.go +++ b/core/state/managed_state_test.go @@ -27,7 +27,7 @@ var addr = common.BytesToAddress([]byte("test")) func create() (*ManagedState, *account) { db, _ := ethdb.NewMemDatabase() - statedb, _ := New(common.Hash{}, db) + statedb, _ := New(common.Hash{}, NewDatabase(db)) ms := ManageState(statedb) ms.StateDB.SetNonce(addr, 100) ms.accounts[addr] = newAccount(ms.StateDB.getStateObject(addr)) diff --git a/core/state/state_object.go b/core/state/state_object.go index dcad9d0681..b2378c69c8 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -62,9 +62,10 @@ func (self Storage) Copy() Storage { // Account values can be accessed and modified through the object. // Finally, call CommitTrie to write the modified storage trie into a database. type stateObject struct { - address common.Address // Ethereum address of this account - data Account - db *StateDB + address common.Address + addrHash common.Hash // hash of ethereum address of the account + data Account + db *StateDB // DB error. // State objects are used by the consensus core and VM which are @@ -74,8 +75,8 @@ type stateObject struct { dbErr error // Write caches. - trie *trie.SecureTrie // storage trie, which becomes non-nil on first access - code Code // contract bytecode, which gets set when code is loaded + trie Trie // storage trie, which becomes non-nil on first access + code Code // contract bytecode, which gets set when code is loaded cachedStorage Storage // Storage entry cache to avoid duplicate reads dirtyStorage Storage // Storage entries that need to be flushed to disk @@ -112,7 +113,15 @@ func newObject(db *StateDB, address common.Address, data Account, onDirty func(a if data.CodeHash == nil { data.CodeHash = emptyCodeHash } - return &stateObject{db: db, address: address, data: data, cachedStorage: make(Storage), dirtyStorage: make(Storage), onDirty: onDirty} + return &stateObject{ + db: db, + address: address, + addrHash: crypto.Keccak256Hash(address[:]), + data: data, + cachedStorage: make(Storage), + dirtyStorage: make(Storage), + onDirty: onDirty, + } } // EncodeRLP implements rlp.Encoder. @@ -148,12 +157,12 @@ func (c *stateObject) touch() { c.touched = true } -func (c *stateObject) getTrie(db trie.Database) *trie.SecureTrie { +func (c *stateObject) getTrie(db Database) Trie { if c.trie == nil { var err error - c.trie, err = trie.NewSecure(c.data.Root, db, 0) + c.trie, err = db.OpenStorageTrie(c.addrHash, c.data.Root) if err != nil { - c.trie, _ = trie.NewSecure(common.Hash{}, db, 0) + c.trie, _ = db.OpenStorageTrie(c.addrHash, common.Hash{}) c.setError(fmt.Errorf("can't create storage trie: %v", err)) } } @@ -161,13 +170,18 @@ func (c *stateObject) getTrie(db trie.Database) *trie.SecureTrie { } // GetState returns a value in account storage. -func (self *stateObject) GetState(db trie.Database, key common.Hash) common.Hash { +func (self *stateObject) GetState(db Database, key common.Hash) common.Hash { value, exists := self.cachedStorage[key] if exists { return value } // Load from DB in case it is missing. - if enc := self.getTrie(db).Get(key[:]); len(enc) > 0 { + enc, err := self.getTrie(db).TryGet(key[:]) + if err != nil { + self.setError(err) + return common.Hash{} + } + if len(enc) > 0 { _, content, _, err := rlp.Split(enc) if err != nil { self.setError(err) @@ -181,7 +195,7 @@ func (self *stateObject) GetState(db trie.Database, key common.Hash) common.Hash } // SetState updates a value in account storage. -func (self *stateObject) SetState(db trie.Database, key, value common.Hash) { +func (self *stateObject) SetState(db Database, key, value common.Hash) { self.db.journal = append(self.db.journal, storageChange{ account: &self.address, key: key, @@ -201,30 +215,30 @@ func (self *stateObject) setState(key, value common.Hash) { } // updateTrie writes cached storage modifications into the object's storage trie. -func (self *stateObject) updateTrie(db trie.Database) *trie.SecureTrie { +func (self *stateObject) updateTrie(db Database) Trie { tr := self.getTrie(db) for key, value := range self.dirtyStorage { delete(self.dirtyStorage, key) if (value == common.Hash{}) { - tr.Delete(key[:]) + self.setError(tr.TryDelete(key[:])) continue } // Encoding []byte cannot fail, ok to ignore the error. v, _ := rlp.EncodeToBytes(bytes.TrimLeft(value[:], "\x00")) - tr.Update(key[:], v) + self.setError(tr.TryUpdate(key[:], v)) } return tr } // UpdateRoot sets the trie root to the current root hash of -func (self *stateObject) updateRoot(db trie.Database) { +func (self *stateObject) updateRoot(db Database) { self.updateTrie(db) self.data.Root = self.trie.Hash() } // CommitTrie the storage trie of the object to dwb. // This updates the trie root. -func (self *stateObject) CommitTrie(db trie.Database, dbw trie.DatabaseWriter) error { +func (self *stateObject) CommitTrie(db Database, dbw trie.DatabaseWriter) error { self.updateTrie(db) if self.dbErr != nil { return self.dbErr @@ -282,9 +296,7 @@ func (c *stateObject) ReturnGas(gas *big.Int) {} func (self *stateObject) deepCopy(db *StateDB, onDirty func(addr common.Address)) *stateObject { stateObject := newObject(db, self.address, self.data, onDirty) if self.trie != nil { - // A shallow copy makes the two tries independent. - cpy := *self.trie - stateObject.trie = &cpy + stateObject.trie = db.db.CopyTrie(self.trie) } stateObject.code = self.code stateObject.dirtyStorage = self.dirtyStorage.Copy() @@ -305,14 +317,14 @@ func (c *stateObject) Address() common.Address { } // Code returns the contract code associated with this object, if any. -func (self *stateObject) Code(db trie.Database) []byte { +func (self *stateObject) Code(db Database) []byte { if self.code != nil { return self.code } if bytes.Equal(self.CodeHash(), emptyCodeHash) { return nil } - code, err := db.Get(self.CodeHash()) + code, err := db.ContractCode(self.addrHash, common.BytesToHash(self.CodeHash())) if err != nil { self.setError(fmt.Errorf("can't load code hash %x: %v", self.CodeHash(), err)) } diff --git a/core/state/state_test.go b/core/state/state_test.go index 3bc63c1486..bbae3685bb 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -21,14 +21,14 @@ import ( "math/big" "testing" - checker "gopkg.in/check.v1" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + checker "gopkg.in/check.v1" ) type StateSuite struct { + db *ethdb.MemDatabase state *StateDB } @@ -48,7 +48,7 @@ func (s *StateSuite) TestDump(c *checker.C) { // write some of them to the trie s.state.updateStateObject(obj1) s.state.updateStateObject(obj2) - s.state.Commit(false) + s.state.CommitTo(s.db, false) // check that dump contains the state objects that are in trie got := string(s.state.Dump()) @@ -87,23 +87,20 @@ func (s *StateSuite) TestDump(c *checker.C) { } func (s *StateSuite) SetUpTest(c *checker.C) { - db, _ := ethdb.NewMemDatabase() - s.state, _ = New(common.Hash{}, db) + s.db, _ = ethdb.NewMemDatabase() + s.state, _ = New(common.Hash{}, NewDatabase(s.db)) } -func TestNull(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, db) - +func (s *StateSuite) TestNull(c *checker.C) { address := common.HexToAddress("0x823140710bf13990e4500136726d8b55") - state.CreateAccount(address) + s.state.CreateAccount(address) //value := common.FromHex("0x823140710bf13990e4500136726d8b55") var value common.Hash - state.SetState(address, common.Hash{}, value) - state.Commit(false) - value = state.GetState(address, common.Hash{}) + s.state.SetState(address, common.Hash{}, value) + s.state.CommitTo(s.db, false) + value = s.state.GetState(address, common.Hash{}) if !common.EmptyHash(value) { - t.Errorf("expected empty hash. got %x", value) + c.Errorf("expected empty hash. got %x", value) } } @@ -129,17 +126,15 @@ func (s *StateSuite) TestSnapshot(c *checker.C) { c.Assert(data1, checker.DeepEquals, res) } -func TestSnapshotEmpty(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, db) - state.RevertToSnapshot(state.Snapshot()) +func (s *StateSuite) TestSnapshotEmpty(c *checker.C) { + s.state.RevertToSnapshot(s.state.Snapshot()) } // use testing instead of checker because checker does not support // printing/logging in tests (-check.vv does not work) func TestSnapshot2(t *testing.T) { db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, db) + state, _ := New(common.Hash{}, NewDatabase(db)) stateobjaddr0 := toAddr([]byte("so0")) stateobjaddr1 := toAddr([]byte("so1")) @@ -160,7 +155,7 @@ func TestSnapshot2(t *testing.T) { so0.deleted = false state.setStateObject(so0) - root, _ := state.Commit(false) + root, _ := state.CommitTo(db, false) state.Reset(root) // and one with deleted == true @@ -182,8 +177,8 @@ func TestSnapshot2(t *testing.T) { so0Restored := state.getStateObject(stateobjaddr0) // Update lazily-loaded values before comparing. - so0Restored.GetState(db, storageaddr) - so0Restored.Code(db) + so0Restored.GetState(state.db, storageaddr) + so0Restored.Code(state.db) // non-deleted is equal (restored) compareStateObjects(so0Restored, so0, t) diff --git a/core/state/statedb.go b/core/state/statedb.go index 05869a0c8a..694374f822 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -26,23 +26,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - lru "github.com/hashicorp/golang-lru" -) - -// Trie cache generation limit after which to evic trie nodes from memory. -var MaxTrieCacheGen = uint16(120) - -const ( - // Number of past tries to keep. This value is chosen such that - // reasonable chain reorg depths will hit an existing trie. - maxPastTries = 12 - - // Number of codehash->size associations to keep. - codeSizeCacheSize = 100000 ) type revision struct { @@ -56,16 +42,21 @@ type revision struct { // * Contracts // * Accounts type StateDB struct { - db ethdb.Database - trie *trie.SecureTrie - pastTries []*trie.SecureTrie - codeSizeCache *lru.Cache + db Database + trie Trie // This map holds 'live' objects, which will get modified while processing a state transition. stateObjects map[common.Address]*stateObject stateObjectsDirty map[common.Address]struct{} stateObjectsDestructed map[common.Address]struct{} + // DB error. + // State objects are used by the consensus core and VM which are + // unable to deal with database-level errors. Any error that occurs + // during a database read is memoized here and will eventually be returned + // by StateDB.Commit. + dbErr error + // The refund counter, also used by state transitioning. refund *big.Int @@ -86,16 +77,14 @@ type StateDB struct { } // Create a new state from a given trie -func New(root common.Hash, db ethdb.Database) (*StateDB, error) { - tr, err := trie.NewSecure(root, db, MaxTrieCacheGen) +func New(root common.Hash, db Database) (*StateDB, error) { + tr, err := db.OpenTrie(root) if err != nil { return nil, err } - csc, _ := lru.New(codeSizeCacheSize) return &StateDB{ db: db, trie: tr, - codeSizeCache: csc, stateObjects: make(map[common.Address]*stateObject), stateObjectsDirty: make(map[common.Address]struct{}), stateObjectsDestructed: make(map[common.Address]struct{}), @@ -105,36 +94,21 @@ func New(root common.Hash, db ethdb.Database) (*StateDB, error) { }, nil } -// New creates a new statedb by reusing any journalled tries to avoid costly -// disk io. -func (self *StateDB) New(root common.Hash) (*StateDB, error) { - self.lock.Lock() - defer self.lock.Unlock() - - tr, err := self.openTrie(root) - if err != nil { - return nil, err +// setError remembers the first non-nil error it is called with. +func (self *StateDB) setError(err error) { + if self.dbErr == nil { + self.dbErr = err } - return &StateDB{ - db: self.db, - trie: tr, - codeSizeCache: self.codeSizeCache, - stateObjects: make(map[common.Address]*stateObject), - stateObjectsDirty: make(map[common.Address]struct{}), - stateObjectsDestructed: make(map[common.Address]struct{}), - refund: new(big.Int), - logs: make(map[common.Hash][]*types.Log), - preimages: make(map[common.Hash][]byte), - }, nil +} + +func (self *StateDB) Error() error { + return self.dbErr } // Reset clears out all emphemeral state objects from the state db, but keeps // the underlying state trie to avoid reloading data for the next operations. func (self *StateDB) Reset(root common.Hash) error { - self.lock.Lock() - defer self.lock.Unlock() - - tr, err := self.openTrie(root) + tr, err := self.db.OpenTrie(root) if err != nil { return err } @@ -149,34 +123,9 @@ func (self *StateDB) Reset(root common.Hash) error { self.logSize = 0 self.preimages = make(map[common.Hash][]byte) self.clearJournalAndRefund() - return nil } -// openTrie creates a trie. It uses an existing trie if one is available -// from the journal if available. -func (self *StateDB) openTrie(root common.Hash) (*trie.SecureTrie, error) { - for i := len(self.pastTries) - 1; i >= 0; i-- { - if self.pastTries[i].Hash() == root { - tr := *self.pastTries[i] - return &tr, nil - } - } - return trie.NewSecure(root, self.db, MaxTrieCacheGen) -} - -func (self *StateDB) pushTrie(t *trie.SecureTrie) { - self.lock.Lock() - defer self.lock.Unlock() - - if len(self.pastTries) >= maxPastTries { - copy(self.pastTries, self.pastTries[1:]) - self.pastTries[len(self.pastTries)-1] = t - } else { - self.pastTries = append(self.pastTries, t) - } -} - func (self *StateDB) AddLog(log *types.Log) { self.journal = append(self.journal, addLogChange{txhash: self.thash}) @@ -254,10 +203,7 @@ func (self *StateDB) GetNonce(addr common.Address) uint64 { func (self *StateDB) GetCode(addr common.Address) []byte { stateObject := self.getStateObject(addr) if stateObject != nil { - code := stateObject.Code(self.db) - key := common.BytesToHash(stateObject.CodeHash()) - self.codeSizeCache.Add(key, len(code)) - return code + return stateObject.Code(self.db) } return nil } @@ -267,13 +213,12 @@ func (self *StateDB) GetCodeSize(addr common.Address) int { if stateObject == nil { return 0 } - key := common.BytesToHash(stateObject.CodeHash()) - if cached, ok := self.codeSizeCache.Get(key); ok { - return cached.(int) + if stateObject.code != nil { + return len(stateObject.code) } - size := len(stateObject.Code(self.db)) - if stateObject.dbErr == nil { - self.codeSizeCache.Add(key, size) + size, err := self.db.ContractCodeSize(stateObject.addrHash, common.BytesToHash(stateObject.CodeHash())) + if err != nil { + self.setError(err) } return size } @@ -296,7 +241,7 @@ func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash { // StorageTrie returns the storage trie of an account. // The return value is a copy and is nil for non-existent accounts. -func (self *StateDB) StorageTrie(a common.Address) *trie.SecureTrie { +func (self *StateDB) StorageTrie(a common.Address) Trie { stateObject := self.getStateObject(a) if stateObject == nil { return nil @@ -394,14 +339,14 @@ func (self *StateDB) updateStateObject(stateObject *stateObject) { if err != nil { panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err)) } - self.trie.Update(addr[:], data) + self.setError(self.trie.TryUpdate(addr[:], data)) } // deleteStateObject removes the given object from the state trie. func (self *StateDB) deleteStateObject(stateObject *stateObject) { stateObject.deleted = true addr := stateObject.Address() - self.trie.Delete(addr[:]) + self.setError(self.trie.TryDelete(addr[:])) } // Retrieve a state object given my the address. Returns nil if not found. @@ -415,8 +360,9 @@ func (self *StateDB) getStateObject(addr common.Address) (stateObject *stateObje } // Load the object from the database. - enc := self.trie.Get(addr[:]) + enc, err := self.trie.TryGet(addr[:]) if len(enc) == 0 { + self.setError(err) return nil } var data Account @@ -512,8 +458,6 @@ func (self *StateDB) Copy() *StateDB { state := &StateDB{ db: self.db, trie: self.trie, - pastTries: self.pastTries, - codeSizeCache: self.codeSizeCache, stateObjects: make(map[common.Address]*stateObject, len(self.stateObjectsDirty)), stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)), stateObjectsDestructed: make(map[common.Address]struct{}, len(self.stateObjectsDestructed)), @@ -636,23 +580,6 @@ func (s *StateDB) DeleteSuicides() { } } -// Commit commits all state changes to the database. -func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) { - root, batch := s.CommitBatch(deleteEmptyObjects) - return root, batch.Write() -} - -// CommitBatch commits all state changes to a write batch but does not -// execute the batch. It is used to validate state changes against -// the root hash stored in a block. -func (s *StateDB) CommitBatch(deleteEmptyObjects bool) (root common.Hash, batch ethdb.Batch) { - batch = s.db.NewBatch() - root, _ = s.CommitTo(batch, deleteEmptyObjects) - - log.Debug("Trie cache stats after commit", "misses", trie.CacheMisses(), "unloads", trie.CacheUnloads()) - return root, batch -} - func (s *StateDB) clearJournalAndRefund() { s.journal = nil s.validRevisions = s.validRevisions[:0] @@ -690,8 +617,6 @@ func (s *StateDB) CommitTo(dbw trie.DatabaseWriter, deleteEmptyObjects bool) (ro } // Write trie changes. root, err = s.trie.CommitTo(dbw) - if err == nil { - s.pushTrie(s.trie) - } + log.Debug("Trie cache stats after commit", "misses", trie.CacheMisses(), "unloads", trie.CacheUnloads()) return root, err } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 72b638f970..b2bd18e65e 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -28,6 +28,8 @@ import ( "testing" "testing/quick" + check "gopkg.in/check.v1" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -38,7 +40,7 @@ import ( func TestUpdateLeaks(t *testing.T) { // Create an empty state database db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, db) + state, _ := New(common.Hash{}, NewDatabase(db)) // Update it with some accounts for i := byte(0); i < 255; i++ { @@ -66,8 +68,8 @@ func TestIntermediateLeaks(t *testing.T) { // Create two state databases, one transitioning to the final state, the other final from the beginning transDb, _ := ethdb.NewMemDatabase() finalDb, _ := ethdb.NewMemDatabase() - transState, _ := New(common.Hash{}, transDb) - finalState, _ := New(common.Hash{}, finalDb) + transState, _ := New(common.Hash{}, NewDatabase(transDb)) + finalState, _ := New(common.Hash{}, NewDatabase(finalDb)) modify := func(state *StateDB, addr common.Address, i, tweak byte) { state.SetBalance(addr, big.NewInt(int64(11*i)+int64(tweak))) @@ -95,10 +97,10 @@ func TestIntermediateLeaks(t *testing.T) { } // Commit and cross check the databases. - if _, err := transState.Commit(false); err != nil { + if _, err := transState.CommitTo(transDb, false); err != nil { t.Fatalf("failed to commit transition state: %v", err) } - if _, err := finalState.Commit(false); err != nil { + if _, err := finalState.CommitTo(finalDb, false); err != nil { t.Fatalf("failed to commit final state: %v", err) } for _, key := range finalDb.Keys() { @@ -282,7 +284,7 @@ func (test *snapshotTest) run() bool { // Run all actions and create snapshots. var ( db, _ = ethdb.NewMemDatabase() - state, _ = New(common.Hash{}, db) + state, _ = New(common.Hash{}, NewDatabase(db)) snapshotRevs = make([]int, len(test.snapshots)) sindex = 0 ) @@ -297,7 +299,7 @@ func (test *snapshotTest) run() bool { // Revert all snapshots in reverse order. Each revert must yield a state // that is equivalent to fresh state with all actions up the snapshot applied. for sindex--; sindex >= 0; sindex-- { - checkstate, _ := New(common.Hash{}, db) + checkstate, _ := New(common.Hash{}, NewDatabase(db)) for _, action := range test.actions[:test.snapshots[sindex]] { action.fn(action, checkstate) } @@ -354,21 +356,19 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error { return nil } -func TestTouchDelete(t *testing.T) { - db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, db) - state.GetOrNewStateObject(common.Address{}) - root, _ := state.Commit(false) - state.Reset(root) +func (s *StateSuite) TestTouchDelete(c *check.C) { + s.state.GetOrNewStateObject(common.Address{}) + root, _ := s.state.CommitTo(s.db, false) + s.state.Reset(root) - snapshot := state.Snapshot() - state.AddBalance(common.Address{}, new(big.Int)) - if len(state.stateObjectsDirty) != 1 { - t.Fatal("expected one dirty state object") + snapshot := s.state.Snapshot() + s.state.AddBalance(common.Address{}, new(big.Int)) + if len(s.state.stateObjectsDirty) != 1 { + c.Fatal("expected one dirty state object") } - state.RevertToSnapshot(snapshot) - if len(state.stateObjectsDirty) != 0 { - t.Fatal("expected no dirty state object") + s.state.RevertToSnapshot(snapshot) + if len(s.state.stateObjectsDirty) != 0 { + c.Fatal("expected no dirty state object") } } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 108ebb3208..06c572ea62 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -36,9 +36,10 @@ type testAccount struct { } // makeTestState create a sample test state to test node-wise reconstruction. -func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { +func makeTestState() (Database, *ethdb.MemDatabase, common.Hash, []*testAccount) { // Create an empty state - db, _ := ethdb.NewMemDatabase() + mem, _ := ethdb.NewMemDatabase() + db := NewDatabase(mem) state, _ := New(common.Hash{}, db) // Fill it with some arbitrary data @@ -60,17 +61,17 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { state.updateStateObject(obj) accounts = append(accounts, acc) } - root, _ := state.Commit(false) + root, _ := state.CommitTo(mem, false) // Return the generated state - return db, root, accounts + return db, mem, root, accounts } // checkStateAccounts cross references a reconstructed state with an expected // account array. func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) { // Check root availability and state contents - state, err := New(root, db) + state, err := New(root, NewDatabase(db)) if err != nil { t.Fatalf("failed to create state trie at %x: %v", root, err) } @@ -90,13 +91,28 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou } } -// checkStateConsistency checks that all nodes in a state trie are indeed present. +// checkTrieConsistency checks that all nodes in a (sub-)trie are indeed present. +func checkTrieConsistency(db ethdb.Database, root common.Hash) error { + if v, _ := db.Get(root[:]); v == nil { + return nil // Consider a non existent state consistent. + } + trie, err := trie.New(root, db) + if err != nil { + return err + } + it := trie.NodeIterator(nil) + for it.Next(true) { + } + return it.Error() +} + +// checkStateConsistency checks that all data of a state root is present. func checkStateConsistency(db ethdb.Database, root common.Hash) error { // Create and iterate a state trie rooted in a sub-node if _, err := db.Get(root.Bytes()); err != nil { - return nil // Consider a non existent state consistent + return nil // Consider a non existent state consistent. } - state, err := New(root, db) + state, err := New(root, NewDatabase(db)) if err != nil { return err } @@ -122,7 +138,7 @@ func TestIterativeStateSyncBatched(t *testing.T) { testIterativeStateSync(t, func testIterativeStateSync(t *testing.T, batch int) { // Create a random state to copy - srcDb, srcRoot, srcAccounts := makeTestState() + _, srcMem, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() @@ -132,7 +148,7 @@ func testIterativeStateSync(t *testing.T, batch int) { for len(queue) > 0 { results := make([]trie.SyncResult, len(queue)) for i, hash := range queue { - data, err := srcDb.Get(hash.Bytes()) + data, err := srcMem.Get(hash.Bytes()) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", hash, err) } @@ -154,7 +170,7 @@ func testIterativeStateSync(t *testing.T, batch int) { // partial results are returned, and the others sent only later. func TestIterativeDelayedStateSync(t *testing.T) { // Create a random state to copy - srcDb, srcRoot, srcAccounts := makeTestState() + _, srcMem, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() @@ -165,7 +181,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { // Sync only half of the scheduled nodes results := make([]trie.SyncResult, len(queue)/2+1) for i, hash := range queue[:len(results)] { - data, err := srcDb.Get(hash.Bytes()) + data, err := srcMem.Get(hash.Bytes()) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", hash, err) } @@ -191,7 +207,7 @@ func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomS func testIterativeRandomStateSync(t *testing.T, batch int) { // Create a random state to copy - srcDb, srcRoot, srcAccounts := makeTestState() + _, srcMem, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() @@ -205,7 +221,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) { // Fetch all the queued nodes in a random order results := make([]trie.SyncResult, 0, len(queue)) for hash := range queue { - data, err := srcDb.Get(hash.Bytes()) + data, err := srcMem.Get(hash.Bytes()) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", hash, err) } @@ -231,7 +247,7 @@ func testIterativeRandomStateSync(t *testing.T, batch int) { // partial results are returned (Even those randomly), others sent only later. func TestIterativeRandomDelayedStateSync(t *testing.T) { // Create a random state to copy - srcDb, srcRoot, srcAccounts := makeTestState() + _, srcMem, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() @@ -247,7 +263,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { for hash := range queue { delete(queue, hash) - data, err := srcDb.Get(hash.Bytes()) + data, err := srcMem.Get(hash.Bytes()) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", hash, err) } @@ -276,7 +292,9 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { // the database. func TestIncompleteStateSync(t *testing.T) { // Create a random state to copy - srcDb, srcRoot, srcAccounts := makeTestState() + _, srcMem, srcRoot, srcAccounts := makeTestState() + + checkTrieConsistency(srcMem, srcRoot) // Create a destination state and sync with the scheduler dstDb, _ := ethdb.NewMemDatabase() @@ -288,7 +306,7 @@ func TestIncompleteStateSync(t *testing.T) { // Fetch a batch of state nodes results := make([]trie.SyncResult, len(queue)) for i, hash := range queue { - data, err := srcDb.Get(hash.Bytes()) + data, err := srcMem.Get(hash.Bytes()) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", hash, err) } @@ -304,21 +322,18 @@ func TestIncompleteStateSync(t *testing.T) { for _, result := range results { added = append(added, result.Hash) } - // Check that all known sub-tries in the synced state is complete - for _, root := range added { - // Skim through the accounts and make sure the root hash is not a code node - codeHash := false + // Check that all known sub-tries added so far are complete or missing entirely. + checkSubtries: + for _, hash := range added { for _, acc := range srcAccounts { - if root == crypto.Keccak256Hash(acc.code) { - codeHash = true - break + if hash == crypto.Keccak256Hash(acc.code) { + continue checkSubtries // skip trie check of code nodes. } } - // If the root is a real trie node, check consistency - if !codeHash { - if err := checkStateConsistency(dstDb, root); err != nil { - t.Fatalf("state inconsistent: %v", err) - } + // Can't use checkStateConsistency here because subtrie keys may have odd + // length and crash in LeafKey. + if err := checkTrieConsistency(dstDb, hash); err != nil { + t.Fatalf("state inconsistent: %v", err) } } // Fetch the next batch to retrieve diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 4e28522e9f..4903bc3ca4 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -44,7 +44,7 @@ func pricedTransaction(nonce uint64, gaslimit, gasprice *big.Int, key *ecdsa.Pri func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) key, _ := crypto.GenerateKey() newPool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) @@ -95,7 +95,7 @@ func TestStateChangeDuringPoolReset(t *testing.T) { key, _ = crypto.GenerateKey() address = crypto.PubkeyToAddress(key.PublicKey) mux = new(event.TypeMux) - statedb, _ = state.New(common.Hash{}, db) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) trigger = false ) @@ -114,7 +114,7 @@ func TestStateChangeDuringPoolReset(t *testing.T) { // a state change between those fetches. stdb := statedb if trigger { - statedb, _ = state.New(common.Hash{}, db) + statedb, _ = state.New(common.Hash{}, state.NewDatabase(db)) // simulate that the new head block included tx0 and tx1 statedb.SetNonce(address, 2) statedb.SetBalance(address, new(big.Int).SetUint64(params.Ether)) @@ -292,7 +292,7 @@ func TestTransactionChainFork(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool.currentState = func() (*state.StateDB, error) { return statedb, nil } currentState, _ := pool.currentState() currentState.AddBalance(addr, big.NewInt(100000000000000)) @@ -318,7 +318,7 @@ func TestTransactionDoubleNonce(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool.currentState = func() (*state.StateDB, error) { return statedb, nil } currentState, _ := pool.currentState() currentState.AddBalance(addr, big.NewInt(100000000000000)) @@ -628,7 +628,7 @@ func TestTransactionQueueGlobalLimiting(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) pool.resetState() @@ -783,7 +783,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) pool.resetState() @@ -835,7 +835,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) pool.resetState() @@ -868,7 +868,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) pool.resetState() @@ -913,7 +913,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { func TestTransactionPoolRepricing(t *testing.T) { // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) pool.resetState() @@ -1006,7 +1006,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) pool.resetState() @@ -1091,7 +1091,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { func TestTransactionReplacement(t *testing.T) { // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) pool.resetState() diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index aa386a995e..44cde4f70a 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -102,7 +102,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { if cfg.State == nil { db, _ := ethdb.NewMemDatabase() - cfg.State, _ = state.New(common.Hash{}, db) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db)) } var ( address = common.StringToAddress("contract") @@ -133,7 +133,7 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, uint64, error) { if cfg.State == nil { db, _ := ethdb.NewMemDatabase() - cfg.State, _ = state.New(common.Hash{}, db) + cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db)) } var ( vmenv = NewEnv(cfg, cfg.State) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 7f40770d21..2c4dc50265 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -95,7 +95,7 @@ func TestExecute(t *testing.T) { func TestCall(t *testing.T) { db, _ := ethdb.NewMemDatabase() - state, _ := state.New(common.Hash{}, db) + state, _ := state.New(common.Hash{}, state.NewDatabase(db)) address := common.HexToAddress("0x0a") state.SetCode(address, []byte{ byte(vm.PUSH1), 10, diff --git a/eth/api.go b/eth/api.go index 81570988c6..0d90759b6c 100644 --- a/eth/api.go +++ b/eth/api.go @@ -637,7 +637,7 @@ func (api *PrivateDebugAPI) StorageRangeAt(ctx context.Context, blockHash common return storageRangeAt(st, keyStart, maxResult), nil } -func storageRangeAt(st *trie.SecureTrie, start []byte, maxResult int) StorageRangeResult { +func storageRangeAt(st state.Trie, start []byte, maxResult int) StorageRangeResult { it := trie.NewIterator(st.NodeIterator(start)) result := StorageRangeResult{Storage: storageMap{}} for i := 0; i < maxResult && it.Next(); i++ { diff --git a/eth/api_backend.go b/eth/api_backend.go index fe108d2720..166b5084d3 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -81,11 +80,11 @@ func (b *EthApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumb return b.eth.blockchain.GetBlockByNumber(uint64(blockNr)), nil } -func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (ethapi.State, *types.Header, error) { +func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) { // Pending state is only known by the miner if blockNr == rpc.PendingBlockNumber { block, state := b.eth.miner.Pending() - return EthApiState{state}, block.Header(), nil + return state, block.Header(), nil } // Otherwise resolve the block number and return its state header, err := b.HeaderByNumber(ctx, blockNr) @@ -93,7 +92,7 @@ func (b *EthApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc. return nil, nil, err } stateDb, err := b.eth.BlockChain().StateAt(header.Root) - return EthApiState{stateDb}, header, err + return stateDb, header, err } func (b *EthApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) { @@ -108,14 +107,12 @@ func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state ethapi.State, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { - statedb := state.(EthApiState).state - from := statedb.GetOrNewStateObject(msg.From()) - from.SetBalance(math.MaxBig256) +func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { + state.SetBalance(msg.From(), math.MaxBig256) vmError := func() error { return nil } context := core.NewEVMContext(msg, header, b.eth.BlockChain(), nil) - return vm.NewEVM(context, statedb, b.eth.chainConfig, vmCfg), vmError, nil + return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), vmError, nil } func (b *EthApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { @@ -200,23 +197,3 @@ func (b *EthApiBackend) EventMux() *event.TypeMux { func (b *EthApiBackend) AccountManager() *accounts.Manager { return b.eth.AccountManager() } - -type EthApiState struct { - state *state.StateDB -} - -func (s EthApiState) GetBalance(ctx context.Context, addr common.Address) (*big.Int, error) { - return s.state.GetBalance(addr), nil -} - -func (s EthApiState) GetCode(ctx context.Context, addr common.Address) ([]byte, error) { - return s.state.GetCode(addr), nil -} - -func (s EthApiState) GetState(ctx context.Context, a common.Address, b common.Hash) (common.Hash, error) { - return s.state.GetState(a, b), nil -} - -func (s EthApiState) GetNonce(ctx context.Context, addr common.Address) (uint64, error) { - return s.state.GetNonce(addr), nil -} diff --git a/eth/api_test.go b/eth/api_test.go index f8d2e9c764..49ce386886 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -32,7 +32,7 @@ func TestStorageRangeAt(t *testing.T) { // Create a state where account 0x010000... has a few storage entries. var ( db, _ = ethdb.NewMemDatabase() - state, _ = state.New(common.Hash{}, db) + state, _ = state.New(common.Hash{}, state.NewDatabase(db)) addr = common.Address{0x01} keys = []common.Hash{ // hashes of Keys of storage common.HexToHash("340dd630ad21bf010b4e676dbfa9ba9a02175262d1fa356232cfde6cb5b47ef2"), diff --git a/eth/bind.go b/eth/bind.go index e5abd86178..0385db1f9f 100644 --- a/eth/bind.go +++ b/eth/bind.go @@ -54,14 +54,12 @@ func NewContractBackend(apiBackend ethapi.Backend) *ContractBackend { // CodeAt retrieves any code associated with the contract from the local API. func (b *ContractBackend) CodeAt(ctx context.Context, contract common.Address, blockNum *big.Int) ([]byte, error) { - out, err := b.bcapi.GetCode(ctx, contract, toBlockNumber(blockNum)) - return common.FromHex(out), err + return b.bcapi.GetCode(ctx, contract, toBlockNumber(blockNum)) } // CodeAt retrieves any code associated with the contract from the local API. func (b *ContractBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) { - out, err := b.bcapi.GetCode(ctx, contract, rpc.PendingBlockNumber) - return common.FromHex(out), err + return b.bcapi.GetCode(ctx, contract, rpc.PendingBlockNumber) } // ContractCall implements bind.ContractCaller executing an Ethereum contract diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 267a0def94..1fb5a0910e 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -657,7 +657,7 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot) } if index > 0 { - if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, tester.stateDb); statedb == nil || err != nil { + if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, state.NewDatabase(tester.stateDb)); statedb == nil || err != nil { t.Fatalf("state reconstruction failed: %v", err) } } diff --git a/eth/handler_test.go b/eth/handler_test.go index 413ed2bff0..ca9c9e1b40 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -374,7 +374,7 @@ func testGetNodeData(t *testing.T, protocol int) { } accounts := []common.Address{testBank, acc1Addr, acc2Addr} for i := uint64(0); i <= pm.blockchain.CurrentBlock().NumberU64(); i++ { - trie, _ := state.New(pm.blockchain.GetBlockByNumber(i).Root(), statedb) + trie, _ := state.New(pm.blockchain.GetBlockByNumber(i).Root(), state.NewDatabase(statedb)) for j, acc := range accounts { state, _ := pm.blockchain.State() diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index da5dc5d583..c22c56dfb3 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -447,8 +447,8 @@ func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Add if state == nil || err != nil { return nil, err } - - return state.GetBalance(ctx, address) + b := state.GetBalance(address) + return b, state.Error() } // GetBlockByNumber returns the requested block. When blockNr is -1 the chain head is returned. When fullTx is true all @@ -529,31 +529,25 @@ func (s *PublicBlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, bloc } // GetCode returns the code stored at the given address in the state for the given block number. -func (s *PublicBlockChainAPI) GetCode(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (string, error) { +func (s *PublicBlockChainAPI) GetCode(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr) if state == nil || err != nil { - return "", err + return nil, err } - res, err := state.GetCode(ctx, address) - if len(res) == 0 || err != nil { // backwards compatibility - return "0x", err - } - return common.ToHex(res), nil + code := state.GetCode(address) + return code, state.Error() } // GetStorageAt returns the storage from the state at the given address, key and // block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta block // numbers are also allowed. -func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNr rpc.BlockNumber) (string, error) { +func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.Address, key string, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr) if state == nil || err != nil { - return "0x", err + return nil, err } - res, err := state.GetState(ctx, address, common.HexToHash(key)) - if err != nil { - return "0x", err - } - return res.Hex(), nil + res := state.GetState(address, common.HexToHash(key)) + return res[:], state.Error() } // callmsg is the message type used for call transitions. @@ -978,11 +972,8 @@ func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, addr if state == nil || err != nil { return nil, err } - nonce, err := state.GetNonce(ctx, address) - if err != nil { - return nil, err - } - return (*hexutil.Uint64)(&nonce), nil + nonce := state.GetNonce(address) + return (*hexutil.Uint64)(&nonce), state.Error() } // getTransactionBlockData fetches the meta data for the given transaction from the chain database. This is useful to diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 68b5069d08..d122b7915b 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" @@ -47,11 +48,12 @@ type Backend interface { SetHead(number uint64) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Block, error) - StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (State, *types.Header, error) + StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetTd(blockHash common.Hash) *big.Int - GetEVM(ctx context.Context, msg core.Message, state State, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) + GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) + // TxPool API SendTx(ctx context.Context, signedTx *types.Transaction) error RemoveTx(txHash common.Hash) @@ -65,13 +67,6 @@ type Backend interface { CurrentBlock() *types.Block } -type State interface { - GetBalance(ctx context.Context, addr common.Address) (*big.Int, error) - GetCode(ctx context.Context, addr common.Address) ([]byte, error) - GetState(ctx context.Context, a common.Address, b common.Hash) (common.Hash, error) - GetNonce(ctx context.Context, addr common.Address) (uint64, error) -} - func GetAPIs(apiBackend Backend) []rpc.API { nonceLock := new(AddrLocker) return []rpc.API{ diff --git a/les/api_backend.go b/les/api_backend.go index 7d69046deb..7a3c2447c9 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -24,13 +24,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" @@ -70,12 +70,12 @@ func (b *LesApiBackend) BlockByNumber(ctx context.Context, blockNr rpc.BlockNumb return b.GetBlock(ctx, header.Hash()) } -func (b *LesApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (ethapi.State, *types.Header, error) { +func (b *LesApiBackend) StateAndHeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*state.StateDB, *types.Header, error) { header, err := b.HeaderByNumber(ctx, blockNr) if header == nil || err != nil { return nil, nil, err } - return light.NewLightState(light.StateTrieID(header), b.eth.odr), header, nil + return light.NewState(ctx, header, b.eth.odr), header, nil } func (b *LesApiBackend) GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) { @@ -90,18 +90,10 @@ func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state ethapi.State, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { - stateDb := state.(*light.LightState).Copy() - addr := msg.From() - from, err := stateDb.GetOrNewStateObject(ctx, addr) - if err != nil { - return nil, nil, err - } - from.SetBalance(math.MaxBig256) - - vmstate := light.NewVMState(ctx, stateDb) +func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { + state.SetBalance(msg.From(), math.MaxBig256) context := core.NewEVMContext(msg, header, b.eth.blockchain, nil) - return vm.NewEVM(context, vmstate, b.eth.chainConfig, vmCfg), vmstate.Error, nil + return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), state.Error, nil } func (b *LesApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { diff --git a/les/odr_test.go b/les/odr_test.go index 7b34996cee..3a0fd6738f 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -75,24 +75,23 @@ func odrAccounts(ctx context.Context, db ethdb.Database, config *params.ChainCon dummyAddr := common.HexToAddress("1234567812345678123456781234567812345678") acc := []common.Address{testBankAddress, acc1Addr, acc2Addr, dummyAddr} - var res []byte + var ( + res []byte + st *state.StateDB + err error + ) for _, addr := range acc { if bc != nil { header := bc.GetHeaderByHash(bhash) - st, err := state.New(header.Root, db) - if err == nil { - bal := st.GetBalance(addr) - rlp, _ := rlp.EncodeToBytes(bal) - res = append(res, rlp...) - } + st, err = state.New(header.Root, state.NewDatabase(db)) } else { header := lc.GetHeaderByHash(bhash) - st := light.NewLightState(light.StateTrieID(header), lc.Odr()) - bal, err := st.GetBalance(ctx, addr) - if err == nil { - rlp, _ := rlp.EncodeToBytes(bal) - res = append(res, rlp...) - } + st = light.NewState(ctx, header, lc.Odr()) + } + if err == nil { + bal := st.GetBalance(addr) + rlp, _ := rlp.EncodeToBytes(bal) + res = append(res, rlp...) } } @@ -115,7 +114,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai data[35] = byte(i) if bc != nil { header := bc.GetHeaderByHash(bhash) - statedb, err := state.New(header.Root, db) + statedb, err := state.New(header.Root, state.NewDatabase(db)) if err == nil { from := statedb.GetOrNewStateObject(testBankAddress) @@ -133,23 +132,15 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai } } else { header := lc.GetHeaderByHash(bhash) - state := light.NewLightState(light.StateTrieID(header), lc.Odr()) - vmstate := light.NewVMState(ctx, state) - from, err := state.GetOrNewStateObject(ctx, testBankAddress) - if err == nil { - from.SetBalance(math.MaxBig256) - - msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), big.NewInt(100000), new(big.Int), data, false)} - - context := core.NewEVMContext(msg, header, lc, nil) - vmenv := vm.NewEVM(context, vmstate, config, vm.Config{}) - - //vmenv := light.NewEnv(ctx, state, config, lc, msg, header, vm.Config{}) - gp := new(core.GasPool).AddGas(math.MaxBig256) - ret, _, _ := core.ApplyMessage(vmenv, msg, gp) - if vmstate.Error() == nil { - res = append(res, ret...) - } + state := light.NewState(ctx, header, lc.Odr()) + state.SetBalance(testBankAddress, math.MaxBig256) + msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), big.NewInt(100000), new(big.Int), data, false)} + context := core.NewEVMContext(msg, header, lc, nil) + vmenv := vm.NewEVM(context, state, config, vm.Config{}) + gp := new(core.GasPool).AddGas(math.MaxBig256) + ret, _, _ := core.ApplyMessage(vmenv, msg, gp) + if state.Error() == nil { + res = append(res, ret...) } } } diff --git a/les/request_test.go b/les/request_test.go index 3add5f20d7..6b594462dc 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -62,7 +62,7 @@ func tfCodeAccess(db ethdb.Database, bhash common.Hash, number uint64) light.Odr return nil } sti := light.StateTrieID(header) - ci := light.StorageTrieID(sti, testContractAddr, common.Hash{}) + ci := light.StorageTrieID(sti, crypto.Keccak256Hash(testContractAddr[:]), common.Hash{}) return &light.CodeRequest{Id: ci, Hash: crypto.Keccak256Hash(testContractCodeDeployed)} } diff --git a/light/lightchain.go b/light/lightchain.go index 5b7e570417..87436f4a57 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -180,11 +180,6 @@ func (self *LightChain) Status() (td *big.Int, currentBlock common.Hash, genesis return self.GetTd(hash, header.Number.Uint64()), hash, self.genesisBlock.Hash() } -// State returns a new mutable state based on the current HEAD block. -func (self *LightChain) State() *LightState { - return NewLightState(StateTrieID(self.hc.CurrentHeader()), self.odr) -} - // Reset purges the entire blockchain, restoring it to its genesis state. func (bc *LightChain) Reset() { bc.ResetWithGenesisBlock(bc.genesisBlock) diff --git a/light/odr.go b/light/odr.go index ca6364f287..d19a488f62 100644 --- a/light/odr.go +++ b/light/odr.go @@ -34,7 +34,7 @@ import ( // service is not required. var NoOdr = context.Background() -// OdrBackend is an interface to a backend service that handles ODR retrievals +// OdrBackend is an interface to a backend service that handles ODR retrievals type type OdrBackend interface { Database() ethdb.Database Retrieve(ctx context.Context, req OdrRequest) error @@ -66,11 +66,11 @@ func StateTrieID(header *types.Header) *TrieID { // StorageTrieID returns a TrieID for a contract storage trie at a given account // of a given state trie. It also requires the root hash of the trie for // checking Merkle proofs. -func StorageTrieID(state *TrieID, addr common.Address, root common.Hash) *TrieID { +func StorageTrieID(state *TrieID, addrHash, root common.Hash) *TrieID { return &TrieID{ BlockHash: state.BlockHash, BlockNumber: state.BlockNumber, - AccKey: crypto.Keccak256(addr[:]), + AccKey: addrHash[:], Root: root, } } @@ -102,7 +102,7 @@ func storeProof(db ethdb.Database, proof []rlp.RawValue) { // CodeRequest is the ODR request type for retrieving contract code type CodeRequest struct { OdrRequest - Id *TrieID + Id *TrieID // references storage trie of the account Hash common.Hash Data []byte } diff --git a/light/odr_test.go b/light/odr_test.go index 576e3abc9a..544b64effc 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -86,11 +86,11 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { return nil } -type odrTestFn func(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) []byte +type odrTestFn func(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) -func TestOdrGetBlockLes1(t *testing.T) { testChainOdr(t, 1, 1, odrGetBlock) } +func TestOdrGetBlockLes1(t *testing.T) { testChainOdr(t, 1, odrGetBlock) } -func odrGetBlock(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) []byte { +func odrGetBlock(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { var block *types.Block if bc != nil { block = bc.GetBlockByHash(bhash) @@ -98,15 +98,15 @@ func odrGetBlock(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc block, _ = lc.GetBlockByHash(ctx, bhash) } if block == nil { - return nil + return nil, nil } rlp, _ := rlp.EncodeToBytes(block) - return rlp + return rlp, nil } -func TestOdrGetReceiptsLes1(t *testing.T) { testChainOdr(t, 1, 1, odrGetReceipts) } +func TestOdrGetReceiptsLes1(t *testing.T) { testChainOdr(t, 1, odrGetReceipts) } -func odrGetReceipts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) []byte { +func odrGetReceipts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { var receipts types.Receipts if bc != nil { receipts = core.GetBlockReceipts(db, bhash, core.GetBlockNumber(db, bhash)) @@ -114,43 +114,37 @@ func odrGetReceipts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, receipts, _ = GetBlockReceipts(ctx, lc.Odr(), bhash, core.GetBlockNumber(db, bhash)) } if receipts == nil { - return nil + return nil, nil } rlp, _ := rlp.EncodeToBytes(receipts) - return rlp + return rlp, nil } -func TestOdrAccountsLes1(t *testing.T) { testChainOdr(t, 1, 1, odrAccounts) } +func TestOdrAccountsLes1(t *testing.T) { testChainOdr(t, 1, odrAccounts) } -func odrAccounts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) []byte { +func odrAccounts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { dummyAddr := common.HexToAddress("1234567812345678123456781234567812345678") acc := []common.Address{testBankAddress, acc1Addr, acc2Addr, dummyAddr} - var res []byte - for _, addr := range acc { - if bc != nil { - header := bc.GetHeaderByHash(bhash) - st, err := state.New(header.Root, db) - if err == nil { - bal := st.GetBalance(addr) - rlp, _ := rlp.EncodeToBytes(bal) - res = append(res, rlp...) - } - } else { - header := lc.GetHeaderByHash(bhash) - st := NewLightState(StateTrieID(header), lc.Odr()) - bal, err := st.GetBalance(ctx, addr) - if err == nil { - rlp, _ := rlp.EncodeToBytes(bal) - res = append(res, rlp...) - } - } + var st *state.StateDB + if bc == nil { + header := lc.GetHeaderByHash(bhash) + st = NewState(ctx, header, lc.Odr()) + } else { + header := bc.GetHeaderByHash(bhash) + st, _ = state.New(header.Root, state.NewDatabase(db)) } - return res + var res []byte + for _, addr := range acc { + bal := st.GetBalance(addr) + rlp, _ := rlp.EncodeToBytes(bal) + res = append(res, rlp...) + } + return res, st.Error() } -func TestOdrContractCallLes1(t *testing.T) { testChainOdr(t, 1, 2, odrContractCall) } +func TestOdrContractCallLes1(t *testing.T) { testChainOdr(t, 1, odrContractCall) } type callmsg struct { types.Message @@ -158,50 +152,42 @@ type callmsg struct { func (callmsg) CheckNonce() bool { return false } -func odrContractCall(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) []byte { +func odrContractCall(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { data := common.Hex2Bytes("60CD26850000000000000000000000000000000000000000000000000000000000000000") - config := params.TestChainConfig var res []byte for i := 0; i < 3; i++ { data[35] = byte(i) - if bc != nil { - header := bc.GetHeaderByHash(bhash) - statedb, err := state.New(header.Root, db) - if err == nil { - from := statedb.GetOrNewStateObject(testBankAddress) - from.SetBalance(math.MaxBig256) - msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), big.NewInt(1000000), new(big.Int), data, false)} - - context := core.NewEVMContext(msg, header, bc, nil) - vmenv := vm.NewEVM(context, statedb, config, vm.Config{}) - - gp := new(core.GasPool).AddGas(math.MaxBig256) - ret, _, _ := core.ApplyMessage(vmenv, msg, gp) - res = append(res, ret...) - } + var ( + st *state.StateDB + header *types.Header + chain core.ChainContext + ) + if bc == nil { + chain = lc + header = lc.GetHeaderByHash(bhash) + st = NewState(ctx, header, lc.Odr()) } else { - header := lc.GetHeaderByHash(bhash) - state := NewLightState(StateTrieID(header), lc.Odr()) - vmstate := NewVMState(ctx, state) - from, err := state.GetOrNewStateObject(ctx, testBankAddress) - if err == nil { - from.SetBalance(math.MaxBig256) + chain = bc + header = bc.GetHeaderByHash(bhash) + st, _ = state.New(header.Root, state.NewDatabase(db)) + } - msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), big.NewInt(1000000), new(big.Int), data, false)} - context := core.NewEVMContext(msg, header, lc, nil) - vmenv := vm.NewEVM(context, vmstate, config, vm.Config{}) - gp := new(core.GasPool).AddGas(math.MaxBig256) - ret, _, _ := core.ApplyMessage(vmenv, msg, gp) - if vmstate.Error() == nil { - res = append(res, ret...) - } - } + // Perform read-only call. + st.SetBalance(testBankAddress, math.MaxBig256) + msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), big.NewInt(1000000), new(big.Int), data, false)} + context := core.NewEVMContext(msg, header, chain, nil) + vmenv := vm.NewEVM(context, st, config, vm.Config{}) + gp := new(core.GasPool).AddGas(math.MaxBig256) + ret, _, _ := core.ApplyMessage(vmenv, msg, gp) + res = append(res, ret...) + if st.Error() != nil { + return res, st.Error() } } - return res + return res, nil } func testChainGen(i int, block *core.BlockGen) { @@ -245,7 +231,7 @@ func testChainGen(i int, block *core.BlockGen) { } } -func testChainOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { +func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { var ( evmux = new(event.TypeMux) sdb, _ = ethdb.NewMemDatabase() @@ -258,46 +244,58 @@ func testChainOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { blockchain, _ := core.NewBlockChain(sdb, params.TestChainConfig, ethash.NewFullFaker(), evmux, vm.Config{}) gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, sdb, 4, testChainGen) if _, err := blockchain.InsertChain(gchain); err != nil { - panic(err) + t.Fatal(err) } odr := &testOdr{sdb: sdb, ldb: ldb} - lightchain, _ := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), evmux) + lightchain, err := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker(), evmux) + if err != nil { + t.Fatal(err) + } headers := make([]*types.Header, len(gchain)) for i, block := range gchain { headers[i] = block.Header() } if _, err := lightchain.InsertHeaderChain(headers, 1); err != nil { - panic(err) + t.Fatal(err) } - test := func(expFail uint64) { + test := func(expFail int) { for i := uint64(0); i <= blockchain.CurrentHeader().Number.Uint64(); i++ { bhash := core.GetCanonicalHash(sdb, i) - b1 := fn(NoOdr, sdb, blockchain, nil, bhash) + b1, err := fn(NoOdr, sdb, blockchain, nil, bhash) + if err != nil { + t.Fatalf("error in full-node test for block %d: %v", i, err) + } ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() - b2 := fn(ctx, ldb, nil, lightchain, bhash) + + exp := i < uint64(expFail) + b2, err := fn(ctx, ldb, nil, lightchain, bhash) + if err != nil && exp { + t.Errorf("error in ODR test for block %d: %v", i, err) + } eq := bytes.Equal(b1, b2) - exp := i < expFail if exp && !eq { - t.Errorf("odr mismatch") - } - if !exp && eq { - t.Errorf("unexpected odr match") + t.Errorf("ODR test output for block %d doesn't match full node", i) } } } - odr.disable = true // expect retrievals to fail (except genesis block) without a les peer - test(expFail) - odr.disable = false - // expect all retrievals to pass - test(5) + t.Log("checking without ODR") odr.disable = true + test(1) + + // expect all retrievals to pass with ODR enabled + t.Log("checking with ODR") + odr.disable = false + test(len(gchain)) + // still expect all retrievals to pass, now data should be cached locally - test(5) + t.Log("checking without ODR, should be cached") + odr.disable = true + test(len(gchain)) } diff --git a/light/odr_util.go b/light/odr_util.go index d7f8458f12..fcdfdb82c6 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -106,25 +106,6 @@ func GetCanonicalHash(ctx context.Context, odr OdrBackend, number uint64) (commo return common.Hash{}, err } -// retrieveContractCode tries to retrieve the contract code of the given account -// with the given hash from the network (id points to the storage trie belonging -// to the same account) -func retrieveContractCode(ctx context.Context, odr OdrBackend, id *TrieID, hash common.Hash) ([]byte, error) { - if hash == sha3_nil { - return nil, nil - } - res, _ := odr.Database().Get(hash[:]) - if res != nil { - return res, nil - } - r := &CodeRequest{Id: id, Hash: hash} - if err := odr.Retrieve(ctx, r); err != nil { - return nil, err - } else { - return r.Data, nil - } -} - // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func GetBodyRLP(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (rlp.RawValue, error) { if data := core.GetBodyRLP(odr.Database(), hash, number); data != nil { diff --git a/light/state.go b/light/state.go deleted file mode 100644 index b184dc3a5c..0000000000 --- a/light/state.go +++ /dev/null @@ -1,316 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "context" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" -) - -// LightState is a memory representation of a state. -// This version is ODR capable, caching only the already accessed part of the -// state, retrieving unknown parts on-demand from the ODR backend. Changes are -// never stored in the local database, only in the memory objects. -type LightState struct { - odr OdrBackend - trie *LightTrie - id *TrieID - stateObjects map[string]*StateObject - refund *big.Int -} - -// NewLightState creates a new LightState with the specified root. -// Note that the creation of a light state is always successful, even if the -// root is non-existent. In that case, ODR retrieval will always be unsuccessful -// and every operation will return with an error or wait for the context to be -// cancelled. -func NewLightState(id *TrieID, odr OdrBackend) *LightState { - var tr *LightTrie - if id != nil { - tr = NewLightTrie(id, odr, true) - } - return &LightState{ - odr: odr, - trie: tr, - id: id, - stateObjects: make(map[string]*StateObject), - refund: new(big.Int), - } -} - -// AddRefund adds an amount to the refund value collected during a vm execution -func (self *LightState) AddRefund(gas *big.Int) { - self.refund.Add(self.refund, gas) -} - -// HasAccount returns true if an account exists at the given address -func (self *LightState) HasAccount(ctx context.Context, addr common.Address) (bool, error) { - so, err := self.GetStateObject(ctx, addr) - return so != nil, err -} - -// GetBalance retrieves the balance from the given address or 0 if the account does -// not exist -func (self *LightState) GetBalance(ctx context.Context, addr common.Address) (*big.Int, error) { - stateObject, err := self.GetStateObject(ctx, addr) - if err != nil { - return common.Big0, err - } - if stateObject != nil { - return stateObject.balance, nil - } - - return common.Big0, nil -} - -// GetNonce returns the nonce at the given address or 0 if the account does -// not exist -func (self *LightState) GetNonce(ctx context.Context, addr common.Address) (uint64, error) { - stateObject, err := self.GetStateObject(ctx, addr) - if err != nil { - return 0, err - } - if stateObject != nil { - return stateObject.nonce, nil - } - return 0, nil -} - -// GetCode returns the contract code at the given address or nil if the account -// does not exist -func (self *LightState) GetCode(ctx context.Context, addr common.Address) ([]byte, error) { - stateObject, err := self.GetStateObject(ctx, addr) - if err != nil { - return nil, err - } - if stateObject != nil { - return stateObject.code, nil - } - return nil, nil -} - -// GetState returns the contract storage value at storage address b from the -// contract address a or common.Hash{} if the account does not exist -func (self *LightState) GetState(ctx context.Context, a common.Address, b common.Hash) (common.Hash, error) { - stateObject, err := self.GetStateObject(ctx, a) - if err == nil && stateObject != nil { - return stateObject.GetState(ctx, b) - } - return common.Hash{}, err -} - -// HasSuicided returns true if the given account has been marked for deletion -// or false if the account does not exist -func (self *LightState) HasSuicided(ctx context.Context, addr common.Address) (bool, error) { - stateObject, err := self.GetStateObject(ctx, addr) - if err == nil && stateObject != nil { - return stateObject.remove, nil - } - return false, err -} - -/* - * SETTERS - */ - -// AddBalance adds the given amount to the balance of the specified account -func (self *LightState) AddBalance(ctx context.Context, addr common.Address, amount *big.Int) error { - stateObject, err := self.GetOrNewStateObject(ctx, addr) - if err == nil && stateObject != nil { - stateObject.AddBalance(amount) - } - return err -} - -// SubBalance adds the given amount to the balance of the specified account -func (self *LightState) SubBalance(ctx context.Context, addr common.Address, amount *big.Int) error { - stateObject, err := self.GetOrNewStateObject(ctx, addr) - if err == nil && stateObject != nil { - stateObject.SubBalance(amount) - } - return err -} - -// SetNonce sets the nonce of the specified account -func (self *LightState) SetNonce(ctx context.Context, addr common.Address, nonce uint64) error { - stateObject, err := self.GetOrNewStateObject(ctx, addr) - if err == nil && stateObject != nil { - stateObject.SetNonce(nonce) - } - return err -} - -// SetCode sets the contract code at the specified account -func (self *LightState) SetCode(ctx context.Context, addr common.Address, code []byte) error { - stateObject, err := self.GetOrNewStateObject(ctx, addr) - if err == nil && stateObject != nil { - stateObject.SetCode(crypto.Keccak256Hash(code), code) - } - return err -} - -// SetState sets the storage value at storage address key of the account addr -func (self *LightState) SetState(ctx context.Context, addr common.Address, key common.Hash, value common.Hash) error { - stateObject, err := self.GetOrNewStateObject(ctx, addr) - if err == nil && stateObject != nil { - stateObject.SetState(key, value) - } - return err -} - -// Delete marks an account to be removed and clears its balance -func (self *LightState) Suicide(ctx context.Context, addr common.Address) (bool, error) { - stateObject, err := self.GetOrNewStateObject(ctx, addr) - if err == nil && stateObject != nil { - stateObject.MarkForDeletion() - stateObject.balance = new(big.Int) - - return true, nil - } - - return false, err -} - -// -// Get, set, new state object methods -// - -// GetStateObject returns the state object of the given account or nil if the -// account does not exist -func (self *LightState) GetStateObject(ctx context.Context, addr common.Address) (stateObject *StateObject, err error) { - stateObject = self.stateObjects[addr.Str()] - if stateObject != nil { - if stateObject.deleted { - stateObject = nil - } - return stateObject, nil - } - data, err := self.trie.Get(ctx, addr[:]) - if err != nil { - return nil, err - } - if len(data) == 0 { - return nil, nil - } - - stateObject, err = DecodeObject(ctx, self.id, addr, self.odr, []byte(data)) - if err != nil { - return nil, err - } - - self.SetStateObject(stateObject) - - return stateObject, nil -} - -// SetStateObject sets the state object of the given account -func (self *LightState) SetStateObject(object *StateObject) { - self.stateObjects[object.Address().Str()] = object -} - -// GetOrNewStateObject returns the state object of the given account or creates a -// new one if the account does not exist -func (self *LightState) GetOrNewStateObject(ctx context.Context, addr common.Address) (*StateObject, error) { - stateObject, err := self.GetStateObject(ctx, addr) - if err == nil && (stateObject == nil || stateObject.deleted) { - stateObject, err = self.CreateStateObject(ctx, addr) - } - return stateObject, err -} - -// newStateObject creates a state object whether it exists in the state or not -func (self *LightState) newStateObject(addr common.Address) *StateObject { - stateObject := NewStateObject(addr, self.odr) - self.stateObjects[addr.Str()] = stateObject - - return stateObject -} - -// CreateStateObject creates creates a new state object and takes ownership. -// This is different from "NewStateObject" -func (self *LightState) CreateStateObject(ctx context.Context, addr common.Address) (*StateObject, error) { - // Get previous (if any) - so, err := self.GetStateObject(ctx, addr) - if err != nil { - return nil, err - } - // Create a new one - newSo := self.newStateObject(addr) - - // If it existed set the balance to the new account - if so != nil { - newSo.balance = so.balance - } - - return newSo, nil -} - -// ForEachStorage calls a callback function for every key/value pair found -// in the local storage cache. Note that unlike core/state.StateObject, -// light.StateObject only returns cached values and doesn't download the -// entire storage tree. -func (self *LightState) ForEachStorage(ctx context.Context, addr common.Address, cb func(key, value common.Hash) bool) error { - so, err := self.GetStateObject(ctx, addr) - if err != nil { - return err - } - - if so == nil { - return nil - } - - for h, v := range so.storage { - cb(h, v) - } - return nil -} - -// -// Setting, copying of the state methods -// - -// Copy creates a copy of the state -func (self *LightState) Copy() *LightState { - // ignore error - we assume state-to-be-copied always exists - state := NewLightState(nil, self.odr) - state.trie = self.trie - state.id = self.id - for k, stateObject := range self.stateObjects { - if stateObject.dirty { - state.stateObjects[k] = stateObject.Copy() - } - } - - state.refund.Set(self.refund) - return state -} - -// Set copies the contents of the given state onto this state, overwriting -// its contents -func (self *LightState) Set(state *LightState) { - self.trie = state.trie - self.stateObjects = state.stateObjects - self.refund = state.refund -} - -// GetRefund returns the refund value collected during a vm execution -func (self *LightState) GetRefund() *big.Int { - return self.refund -} diff --git a/light/state_object.go b/light/state_object.go deleted file mode 100644 index a54ea1d9ff..0000000000 --- a/light/state_object.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "bytes" - "context" - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/rlp" -) - -var emptyCodeHash = crypto.Keccak256(nil) - -// Code represents a contract code in binary form -type Code []byte - -// String returns a string representation of the code -func (self Code) String() string { - return string(self) //strings.Join(Disassemble(self), " ") -} - -// Storage is a memory map cache of a contract storage -type Storage map[common.Hash]common.Hash - -// String returns a string representation of the storage cache -func (self Storage) String() (str string) { - for key, value := range self { - str += fmt.Sprintf("%X : %X\n", key, value) - } - - return -} - -// Copy copies the contents of a storage cache -func (self Storage) Copy() Storage { - cpy := make(Storage) - for key, value := range self { - cpy[key] = value - } - - return cpy -} - -// StateObject is a memory representation of an account or contract and its storage. -// This version is ODR capable, caching only the already accessed part of the -// storage, retrieving unknown parts on-demand from the ODR backend. Changes are -// never stored in the local database, only in the memory objects. -type StateObject struct { - odr OdrBackend - trie *LightTrie - - // Address belonging to this account - address common.Address - // The balance of the account - balance *big.Int - // The nonce of the account - nonce uint64 - // The code hash if code is present (i.e. a contract) - codeHash []byte - // The code for this account - code Code - // Cached storage (flushed when updated) - storage Storage - - // Mark for deletion - // When an object is marked for deletion it will be delete from the trie - // during the "update" phase of the state transition - remove bool - deleted bool - dirty bool -} - -// NewStateObject creates a new StateObject of the specified account address -func NewStateObject(address common.Address, odr OdrBackend) *StateObject { - object := &StateObject{ - odr: odr, - address: address, - balance: new(big.Int), - dirty: true, - codeHash: emptyCodeHash, - storage: make(Storage), - } - object.trie = NewLightTrie(&TrieID{}, odr, true) - return object -} - -// MarkForDeletion marks an account to be removed -func (self *StateObject) MarkForDeletion() { - self.remove = true - self.dirty = true -} - -// getAddr gets the storage value at the given address from the trie -func (c *StateObject) getAddr(ctx context.Context, addr common.Hash) (common.Hash, error) { - var ret []byte - val, err := c.trie.Get(ctx, addr[:]) - if err != nil { - return common.Hash{}, err - } - rlp.DecodeBytes(val, &ret) - return common.BytesToHash(ret), nil -} - -// Storage returns the storage cache object of the account -func (self *StateObject) Storage() Storage { - return self.storage -} - -// GetState returns the storage value at the given address from either the cache -// or the trie -func (self *StateObject) GetState(ctx context.Context, key common.Hash) (common.Hash, error) { - value, exists := self.storage[key] - if !exists { - var err error - value, err = self.getAddr(ctx, key) - if err != nil { - return common.Hash{}, err - } - if (value != common.Hash{}) { - self.storage[key] = value - } - } - - return value, nil -} - -// SetState sets the storage value at the given address -func (self *StateObject) SetState(k, value common.Hash) { - self.storage[k] = value - self.dirty = true -} - -// AddBalance adds the given amount to the account balance -func (c *StateObject) AddBalance(amount *big.Int) { - c.SetBalance(new(big.Int).Add(c.balance, amount)) -} - -// SubBalance subtracts the given amount from the account balance -func (c *StateObject) SubBalance(amount *big.Int) { - c.SetBalance(new(big.Int).Sub(c.balance, amount)) -} - -// SetBalance sets the account balance to the given amount -func (c *StateObject) SetBalance(amount *big.Int) { - c.balance = amount - c.dirty = true -} - -// ReturnGas returns the gas back to the origin. Used by the Virtual machine or Closures -func (c *StateObject) ReturnGas(gas *big.Int) {} - -// Copy creates a copy of the state object -func (self *StateObject) Copy() *StateObject { - stateObject := NewStateObject(self.Address(), self.odr) - stateObject.balance.Set(self.balance) - stateObject.codeHash = common.CopyBytes(self.codeHash) - stateObject.nonce = self.nonce - stateObject.trie = self.trie - stateObject.code = self.code - stateObject.storage = self.storage.Copy() - stateObject.remove = self.remove - stateObject.dirty = self.dirty - stateObject.deleted = self.deleted - - return stateObject -} - -// -// Attribute accessors -// - -// empty returns whether the account is considered empty. -func (self *StateObject) empty() bool { - return self.nonce == 0 && self.balance.Sign() == 0 && bytes.Equal(self.codeHash, emptyCodeHash) -} - -// Balance returns the account balance -func (self *StateObject) Balance() *big.Int { - return self.balance -} - -// Address returns the address of the contract/account -func (self *StateObject) Address() common.Address { - return self.address -} - -// Code returns the contract code -func (self *StateObject) Code() []byte { - return self.code -} - -// SetCode sets the contract code -func (self *StateObject) SetCode(hash common.Hash, code []byte) { - self.code = code - self.codeHash = hash[:] - self.dirty = true -} - -// SetNonce sets the account nonce -func (self *StateObject) SetNonce(nonce uint64) { - self.nonce = nonce - self.dirty = true -} - -// Nonce returns the account nonce -func (self *StateObject) Nonce() uint64 { - return self.nonce -} - -// ForEachStorage calls a callback function for every key/value pair found -// in the local storage cache. Note that unlike core/state.StateObject, -// light.StateObject only returns cached values and doesn't download the -// entire storage tree. -func (self *StateObject) ForEachStorage(cb func(key, value common.Hash) bool) { - for h, v := range self.storage { - cb(h, v) - } -} - -// Never called, but must be present to allow StateObject to be used -// as a vm.Account interface that also satisfies the vm.ContractRef -// interface. Interfaces are awesome. -func (self *StateObject) Value() *big.Int { - panic("Value on StateObject should never be called") -} - -// Encoding - -type extStateObject struct { - Nonce uint64 - Balance *big.Int - Root common.Hash - CodeHash []byte -} - -// DecodeObject decodes an RLP-encoded state object. -func DecodeObject(ctx context.Context, stateID *TrieID, address common.Address, odr OdrBackend, data []byte) (*StateObject, error) { - var ( - obj = &StateObject{address: address, odr: odr, storage: make(Storage)} - ext extStateObject - err error - ) - if err = rlp.DecodeBytes(data, &ext); err != nil { - return nil, err - } - trieID := StorageTrieID(stateID, address, ext.Root) - obj.trie = NewLightTrie(trieID, odr, true) - if !bytes.Equal(ext.CodeHash, emptyCodeHash) { - if obj.code, err = retrieveContractCode(ctx, obj.odr, trieID, common.BytesToHash(ext.CodeHash)); err != nil { - return nil, fmt.Errorf("can't find code for hash %x: %v", ext.CodeHash, err) - } - } - obj.nonce = ext.Nonce - obj.balance = ext.Balance - obj.codeHash = ext.CodeHash - return obj, nil -} diff --git a/light/state_test.go b/light/state_test.go deleted file mode 100644 index e776efec8e..0000000000 --- a/light/state_test.go +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "bytes" - "context" - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" -) - -func makeTestState() (common.Hash, ethdb.Database) { - sdb, _ := ethdb.NewMemDatabase() - st, _ := state.New(common.Hash{}, sdb) - for i := byte(0); i < 100; i++ { - addr := common.Address{i} - for j := byte(0); j < 100; j++ { - st.SetState(addr, common.Hash{j}, common.Hash{i, j}) - } - st.SetNonce(addr, 100) - st.AddBalance(addr, big.NewInt(int64(i))) - st.SetCode(addr, []byte{i, i, i}) - } - root, _ := st.Commit(false) - return root, sdb -} - -func TestLightStateOdr(t *testing.T) { - root, sdb := makeTestState() - header := &types.Header{Root: root, Number: big.NewInt(0)} - core.WriteHeader(sdb, header) - ldb, _ := ethdb.NewMemDatabase() - odr := &testOdr{sdb: sdb, ldb: ldb} - ls := NewLightState(StateTrieID(header), odr) - ctx := context.Background() - - for i := byte(0); i < 100; i++ { - addr := common.Address{i} - err := ls.AddBalance(ctx, addr, big.NewInt(1000)) - if err != nil { - t.Fatalf("Error adding balance to acc[%d]: %v", i, err) - } - err = ls.SetState(ctx, addr, common.Hash{100}, common.Hash{i, 100}) - if err != nil { - t.Fatalf("Error setting storage of acc[%d]: %v", i, err) - } - } - - addr := common.Address{100} - _, err := ls.CreateStateObject(ctx, addr) - if err != nil { - t.Fatalf("Error creating state object: %v", err) - } - err = ls.SetCode(ctx, addr, []byte{100, 100, 100}) - if err != nil { - t.Fatalf("Error setting code: %v", err) - } - err = ls.AddBalance(ctx, addr, big.NewInt(1100)) - if err != nil { - t.Fatalf("Error adding balance to acc[100]: %v", err) - } - for j := byte(0); j < 101; j++ { - err = ls.SetState(ctx, addr, common.Hash{j}, common.Hash{100, j}) - if err != nil { - t.Fatalf("Error setting storage of acc[100]: %v", err) - } - } - err = ls.SetNonce(ctx, addr, 100) - if err != nil { - t.Fatalf("Error setting nonce for acc[100]: %v", err) - } - - for i := byte(0); i < 101; i++ { - addr := common.Address{i} - - bal, err := ls.GetBalance(ctx, addr) - if err != nil { - t.Fatalf("Error getting balance of acc[%d]: %v", i, err) - } - if bal.Int64() != int64(i)+1000 { - t.Fatalf("Incorrect balance at acc[%d]: expected %v, got %v", i, int64(i)+1000, bal.Int64()) - } - - nonce, err := ls.GetNonce(ctx, addr) - if err != nil { - t.Fatalf("Error getting nonce of acc[%d]: %v", i, err) - } - if nonce != 100 { - t.Fatalf("Incorrect nonce at acc[%d]: expected %v, got %v", i, 100, nonce) - } - - code, err := ls.GetCode(ctx, addr) - exp := []byte{i, i, i} - if err != nil { - t.Fatalf("Error getting code of acc[%d]: %v", i, err) - } - if !bytes.Equal(code, exp) { - t.Fatalf("Incorrect code at acc[%d]: expected %v, got %v", i, exp, code) - } - - for j := byte(0); j < 101; j++ { - exp := common.Hash{i, j} - val, err := ls.GetState(ctx, addr, common.Hash{j}) - if err != nil { - t.Fatalf("Error retrieving acc[%d].storage[%d]: %v", i, j, err) - } - if val != exp { - t.Fatalf("Retrieved wrong value from acc[%d].storage[%d]: expected %04x, got %04x", i, j, exp, val) - } - } - } -} - -func TestLightStateSetCopy(t *testing.T) { - root, sdb := makeTestState() - header := &types.Header{Root: root, Number: big.NewInt(0)} - core.WriteHeader(sdb, header) - ldb, _ := ethdb.NewMemDatabase() - odr := &testOdr{sdb: sdb, ldb: ldb} - ls := NewLightState(StateTrieID(header), odr) - ctx := context.Background() - - for i := byte(0); i < 100; i++ { - addr := common.Address{i} - err := ls.AddBalance(ctx, addr, big.NewInt(1000)) - if err != nil { - t.Fatalf("Error adding balance to acc[%d]: %v", i, err) - } - err = ls.SetState(ctx, addr, common.Hash{100}, common.Hash{i, 100}) - if err != nil { - t.Fatalf("Error setting storage of acc[%d]: %v", i, err) - } - } - - ls2 := ls.Copy() - - for i := byte(0); i < 100; i++ { - addr := common.Address{i} - err := ls2.AddBalance(ctx, addr, big.NewInt(1000)) - if err != nil { - t.Fatalf("Error adding balance to acc[%d]: %v", i, err) - } - err = ls2.SetState(ctx, addr, common.Hash{100}, common.Hash{i, 200}) - if err != nil { - t.Fatalf("Error setting storage of acc[%d]: %v", i, err) - } - } - - lsx := ls.Copy() - ls.Set(ls2) - ls2.Set(lsx) - - for i := byte(0); i < 100; i++ { - addr := common.Address{i} - // check balance in ls - bal, err := ls.GetBalance(ctx, addr) - if err != nil { - t.Fatalf("Error getting balance to acc[%d]: %v", i, err) - } - if bal.Int64() != int64(i)+2000 { - t.Fatalf("Incorrect balance at ls.acc[%d]: expected %v, got %v", i, int64(i)+1000, bal.Int64()) - } - // check balance in ls2 - bal, err = ls2.GetBalance(ctx, addr) - if err != nil { - t.Fatalf("Error getting balance to acc[%d]: %v", i, err) - } - if bal.Int64() != int64(i)+1000 { - t.Fatalf("Incorrect balance at ls.acc[%d]: expected %v, got %v", i, int64(i)+1000, bal.Int64()) - } - // check storage in ls - exp := common.Hash{i, 200} - val, err := ls.GetState(ctx, addr, common.Hash{100}) - if err != nil { - t.Fatalf("Error retrieving acc[%d].storage[100]: %v", i, err) - } - if val != exp { - t.Fatalf("Retrieved wrong value from acc[%d].storage[100]: expected %04x, got %04x", i, exp, val) - } - // check storage in ls2 - exp = common.Hash{i, 100} - val, err = ls2.GetState(ctx, addr, common.Hash{100}) - if err != nil { - t.Fatalf("Error retrieving acc[%d].storage[100]: %v", i, err) - } - if val != exp { - t.Fatalf("Retrieved wrong value from acc[%d].storage[100]: expected %04x, got %04x", i, exp, val) - } - } -} - -func TestLightStateDelete(t *testing.T) { - root, sdb := makeTestState() - header := &types.Header{Root: root, Number: big.NewInt(0)} - core.WriteHeader(sdb, header) - ldb, _ := ethdb.NewMemDatabase() - odr := &testOdr{sdb: sdb, ldb: ldb} - ls := NewLightState(StateTrieID(header), odr) - ctx := context.Background() - - addr := common.Address{42} - - b, err := ls.HasAccount(ctx, addr) - if err != nil { - t.Fatalf("HasAccount error: %v", err) - } - if !b { - t.Fatalf("HasAccount returned false, expected true") - } - - b, err = ls.HasSuicided(ctx, addr) - if err != nil { - t.Fatalf("HasSuicided error: %v", err) - } - if b { - t.Fatalf("HasSuicided returned true, expected false") - } - - ls.Suicide(ctx, addr) - - b, err = ls.HasSuicided(ctx, addr) - if err != nil { - t.Fatalf("HasSuicided error: %v", err) - } - if !b { - t.Fatalf("HasSuicided returned false, expected true") - } -} diff --git a/light/trie.go b/light/trie.go index 2988a16cf3..7502b6e5dd 100644 --- a/light/trie.go +++ b/light/trie.go @@ -18,99 +18,216 @@ package light import ( "context" + "fmt" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" ) -// LightTrie is an ODR-capable wrapper around trie.SecureTrie -type LightTrie struct { - trie *trie.SecureTrie - id *TrieID - odr OdrBackend - db ethdb.Database +func NewState(ctx context.Context, head *types.Header, odr OdrBackend) *state.StateDB { + state, _ := state.New(head.Root, NewStateDatabase(ctx, head, odr)) + return state } -// NewLightTrie creates a new LightTrie instance. It doesn't instantly try to -// access the db or network and retrieve the root node, it only initializes its -// encapsulated SecureTrie at the first actual operation. -func NewLightTrie(id *TrieID, odr OdrBackend, useFakeMap bool) *LightTrie { - return &LightTrie{ - // SecureTrie is initialized before first request - id: id, - odr: odr, - db: odr.Database(), +func NewStateDatabase(ctx context.Context, head *types.Header, odr OdrBackend) state.Database { + return &odrDatabase{ctx, StateTrieID(head), odr} +} + +type odrDatabase struct { + ctx context.Context + id *TrieID + backend OdrBackend +} + +func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { + return &odrTrie{db: db, id: db.id}, nil +} + +func (db *odrDatabase) OpenStorageTrie(addrHash, root common.Hash) (state.Trie, error) { + return &odrTrie{db: db, id: StorageTrieID(db.id, addrHash, root)}, nil +} + +func (db *odrDatabase) CopyTrie(t state.Trie) state.Trie { + switch t := t.(type) { + case *odrTrie: + cpy := &odrTrie{db: t.db, id: t.id} + if t.trie != nil { + cpytrie := *t.trie + cpy.trie = &cpytrie + } + return cpy + default: + panic(fmt.Errorf("unknown trie type %T", t)) } } -// retrieveKey retrieves a single key, returns true and stores nodes in local -// database if successful -func (t *LightTrie) retrieveKey(ctx context.Context, key []byte) bool { - r := &TrieRequest{Id: t.id, Key: crypto.Keccak256(key)} - return t.odr.Retrieve(ctx, r) == nil +func (db *odrDatabase) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { + if codeHash == sha3_nil { + return nil, nil + } + if code, err := db.backend.Database().Get(codeHash[:]); err == nil { + return code, nil + } + id := *db.id + id.AccKey = addrHash[:] + req := &CodeRequest{Id: &id, Hash: codeHash} + err := db.backend.Retrieve(db.ctx, req) + return req.Data, err +} + +func (db *odrDatabase) ContractCodeSize(addrHash, codeHash common.Hash) (int, error) { + code, err := db.ContractCode(addrHash, codeHash) + return len(code), err +} + +type odrTrie struct { + db *odrDatabase + id *TrieID + trie *trie.Trie +} + +func (t *odrTrie) TryGet(key []byte) ([]byte, error) { + key = crypto.Keccak256(key) + var res []byte + err := t.do(key, func() (err error) { + res, err = t.trie.TryGet(key) + return err + }) + return res, err +} + +func (t *odrTrie) TryUpdate(key, value []byte) error { + key = crypto.Keccak256(key) + return t.do(key, func() error { + return t.trie.TryDelete(key) + }) +} + +func (t *odrTrie) TryDelete(key []byte) error { + key = crypto.Keccak256(key) + return t.do(key, func() error { + return t.trie.TryDelete(key) + }) +} + +func (t *odrTrie) CommitTo(db trie.DatabaseWriter) (common.Hash, error) { + if t.trie == nil { + return t.id.Root, nil + } + return t.trie.CommitTo(db) +} + +func (t *odrTrie) Hash() common.Hash { + if t.trie == nil { + return t.id.Root + } + return t.trie.Hash() +} + +func (t *odrTrie) NodeIterator(startkey []byte) trie.NodeIterator { + return newNodeIterator(t, startkey) +} + +func (t *odrTrie) GetKey(sha []byte) []byte { + return nil } // do tries and retries to execute a function until it returns with no error or // an error type other than MissingNodeError -func (t *LightTrie) do(ctx context.Context, key []byte, fn func() error) error { - err := fn() - for err != nil { +func (t *odrTrie) do(key []byte, fn func() error) error { + for { + var err error + if t.trie == nil { + t.trie, err = trie.New(t.id.Root, t.db.backend.Database()) + } + if err == nil { + err = fn() + } if _, ok := err.(*trie.MissingNodeError); !ok { return err } - if !t.retrieveKey(ctx, key) { - break + r := &TrieRequest{Id: t.id, Key: key} + if err := t.db.backend.Retrieve(t.db.ctx, r); err != nil { + return fmt.Errorf("can't fetch trie key %x: %v", key, err) } - err = fn() } - return err } -// Get returns the value for key stored in the trie. -// The value bytes must not be modified by the caller. -func (t *LightTrie) Get(ctx context.Context, key []byte) (res []byte, err error) { - err = t.do(ctx, key, func() (err error) { - if t.trie == nil { - t.trie, err = trie.NewSecure(t.id.Root, t.db, 0) - } - if err == nil { - res, err = t.trie.TryGet(key) - } - return - }) - return +type nodeIterator struct { + trie.NodeIterator + t *odrTrie + err error } -// Update associates key with value in the trie. Subsequent calls to -// Get will return value. If value has length zero, any existing value -// is deleted from the trie and calls to Get will return nil. -// -// The value bytes must not be modified by the caller while they are -// stored in the trie. -func (t *LightTrie) Update(ctx context.Context, key, value []byte) (err error) { - err = t.do(ctx, key, func() (err error) { - if t.trie == nil { - t.trie, err = trie.NewSecure(t.id.Root, t.db, 0) - } - if err == nil { - err = t.trie.TryUpdate(key, value) - } - return +func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { + it := &nodeIterator{t: t} + // Open the actual non-ODR trie if that hasn't happened yet. + if t.trie == nil { + it.do(func() error { + t, err := trie.New(t.id.Root, t.db.backend.Database()) + if err == nil { + it.t.trie = t + } + return err + }) + } + it.do(func() error { + it.NodeIterator = it.t.trie.NodeIterator(startkey) + return it.NodeIterator.Error() }) - return + return it } -// Delete removes any existing value for key from the trie. -func (t *LightTrie) Delete(ctx context.Context, key []byte) (err error) { - err = t.do(ctx, key, func() (err error) { - if t.trie == nil { - t.trie, err = trie.NewSecure(t.id.Root, t.db, 0) - } - if err == nil { - err = t.trie.TryDelete(key) - } - return +func (it *nodeIterator) Next(descend bool) bool { + var ok bool + it.do(func() error { + ok = it.NodeIterator.Next(descend) + return it.NodeIterator.Error() }) - return + return ok +} + +// do runs fn and attempts to fill in missing nodes by retrieving. +func (it *nodeIterator) do(fn func() error) { + var lasthash common.Hash + for { + it.err = fn() + missing, ok := it.err.(*trie.MissingNodeError) + if !ok { + return + } + if missing.NodeHash == lasthash { + it.err = fmt.Errorf("retrieve loop for trie node %x", missing.NodeHash) + return + } + lasthash = missing.NodeHash + r := &TrieRequest{Id: it.t.id, Key: nibblesToKey(missing.Path)} + if it.err = it.t.db.backend.Retrieve(it.t.db.ctx, r); it.err != nil { + return + } + } +} + +func (it *nodeIterator) Error() error { + if it.err != nil { + return it.err + } + return it.NodeIterator.Error() +} + +func nibblesToKey(nib []byte) []byte { + if len(nib) > 0 && nib[len(nib)-1] == 0x10 { + nib = nib[:len(nib)-1] // drop terminator + } + if len(nib)&1 == 1 { + nib = append(nib, 0) // make even + } + key := make([]byte, len(nib)/2) + for bi, ni := 0, 0; ni < len(nib); bi, ni = bi+1, ni+2 { + key[bi] = nib[ni]<<4 | nib[ni+1] + } + return key } diff --git a/light/trie_test.go b/light/trie_test.go new file mode 100644 index 0000000000..9b2cf7c2b1 --- /dev/null +++ b/light/trie_test.go @@ -0,0 +1,83 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package light + +import ( + "bytes" + "context" + "fmt" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" +) + +func TestNodeIterator(t *testing.T) { + var ( + fulldb, _ = ethdb.NewMemDatabase() + lightdb, _ = ethdb.NewMemDatabase() + gspec = core.Genesis{Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}} + genesis = gspec.MustCommit(fulldb) + ) + gspec.MustCommit(lightdb) + blockchain, _ := core.NewBlockChain(fulldb, params.TestChainConfig, ethash.NewFullFaker(), new(event.TypeMux), vm.Config{}) + gchain, _ := core.GenerateChain(params.TestChainConfig, genesis, fulldb, 4, testChainGen) + if _, err := blockchain.InsertChain(gchain); err != nil { + panic(err) + } + + ctx := context.Background() + odr := &testOdr{sdb: fulldb, ldb: lightdb} + head := blockchain.CurrentHeader() + lightTrie, _ := NewStateDatabase(ctx, head, odr).OpenTrie(head.Root) + fullTrie, _ := state.NewDatabase(fulldb).OpenTrie(head.Root) + if err := diffTries(fullTrie, lightTrie); err != nil { + t.Fatal(err) + } +} + +func diffTries(t1, t2 state.Trie) error { + i1 := trie.NewIterator(t1.NodeIterator(nil)) + i2 := trie.NewIterator(t2.NodeIterator(nil)) + for i1.Next() && i2.Next() { + if !bytes.Equal(i1.Key, i2.Key) { + spew.Dump(i2) + return fmt.Errorf("tries have different keys %x, %x", i1.Key, i2.Key) + } + if !bytes.Equal(i2.Value, i2.Value) { + return fmt.Errorf("tries differ at key %x", i1.Key) + } + } + switch { + case i1.Err != nil: + return fmt.Errorf("full trie iterator error: %v", i1.Err) + case i2.Err != nil: + return fmt.Errorf("light trie iterator error: %v", i1.Err) + case i1.Next(): + return fmt.Errorf("full trie iterator has more k/v pairs") + case i2.Next(): + return fmt.Errorf("light trie iterator has more k/v pairs") + } + return nil +} diff --git a/light/txpool.go b/light/txpool.go index 7276874b8a..0430b280f6 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -100,17 +101,18 @@ func NewTxPool(config *params.ChainConfig, eventMux *event.TypeMux, chain *Light } // currentState returns the light state of the current head header -func (pool *TxPool) currentState() *LightState { - return NewLightState(StateTrieID(pool.chain.CurrentHeader()), pool.odr) +func (pool *TxPool) currentState(ctx context.Context) *state.StateDB { + return NewState(ctx, pool.chain.CurrentHeader(), pool.odr) } // GetNonce returns the "pending" nonce of a given address. It always queries // the nonce belonging to the latest header too in order to detect if another // client using the same key sent a transaction. func (pool *TxPool) GetNonce(ctx context.Context, addr common.Address) (uint64, error) { - nonce, err := pool.currentState().GetNonce(ctx, addr) - if err != nil { - return 0, err + state := pool.currentState(ctx) + nonce := state.GetNonce(addr) + if state.Error() != nil { + return 0, state.Error() } sn, ok := pool.nonce[addr] if ok && sn > nonce { @@ -357,13 +359,9 @@ func (pool *TxPool) validateTx(ctx context.Context, tx *types.Transaction) error return core.ErrInvalidSender } // Last but not least check for nonce errors - currentState := pool.currentState() - if n, err := currentState.GetNonce(ctx, from); err == nil { - if n > tx.Nonce() { - return core.ErrNonceTooLow - } - } else { - return err + currentState := pool.currentState(ctx) + if n := currentState.GetNonce(from); n > tx.Nonce() { + return core.ErrNonceTooLow } // Check the transaction doesn't exceed the current @@ -382,12 +380,8 @@ func (pool *TxPool) validateTx(ctx context.Context, tx *types.Transaction) error // Transactor should have enough funds to cover the costs // cost == V + GP * GL - if b, err := currentState.GetBalance(ctx, from); err == nil { - if b.Cmp(tx.Cost()) < 0 { - return core.ErrInsufficientFunds - } - } else { - return err + if b := currentState.GetBalance(from); b.Cmp(tx.Cost()) < 0 { + return core.ErrInsufficientFunds } // Should supply enough intrinsic gas @@ -395,7 +389,7 @@ func (pool *TxPool) validateTx(ctx context.Context, tx *types.Transaction) error return core.ErrIntrinsicGas } - return nil + return currentState.Error() } // add validates a new transaction and sets its state pending if processable. diff --git a/light/vm_env.go b/light/vm_env.go deleted file mode 100644 index 54aa12875b..0000000000 --- a/light/vm_env.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "context" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" -) - -// VMState is a wrapper for the light state that holds the actual context and -// passes it to any state operation that requires it. -type VMState struct { - ctx context.Context - state *LightState - snapshots []*LightState - err error -} - -func NewVMState(ctx context.Context, state *LightState) *VMState { - return &VMState{ctx: ctx, state: state} -} - -func (s *VMState) Error() error { - return s.err -} - -func (s *VMState) AddLog(log *types.Log) {} - -func (s *VMState) AddPreimage(hash common.Hash, preimage []byte) {} - -// errHandler handles and stores any state error that happens during execution. -func (s *VMState) errHandler(err error) { - if err != nil && s.err == nil { - s.err = err - } -} - -func (self *VMState) Snapshot() int { - self.snapshots = append(self.snapshots, self.state.Copy()) - return len(self.snapshots) - 1 -} - -func (self *VMState) RevertToSnapshot(idx int) { - self.state.Set(self.snapshots[idx]) - self.snapshots = self.snapshots[:idx] -} - -// CreateAccount creates creates a new account object and takes ownership. -func (s *VMState) CreateAccount(addr common.Address) { - _, err := s.state.CreateStateObject(s.ctx, addr) - s.errHandler(err) -} - -// AddBalance adds the given amount to the balance of the specified account -func (s *VMState) AddBalance(addr common.Address, amount *big.Int) { - err := s.state.AddBalance(s.ctx, addr, amount) - s.errHandler(err) -} - -// SubBalance adds the given amount to the balance of the specified account -func (s *VMState) SubBalance(addr common.Address, amount *big.Int) { - err := s.state.SubBalance(s.ctx, addr, amount) - s.errHandler(err) -} - -// ForEachStorage calls a callback function for every key/value pair found -// in the local storage cache. Note that unlike core/state.StateObject, -// light.StateObject only returns cached values and doesn't download the -// entire storage tree. -func (s *VMState) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) { - err := s.state.ForEachStorage(s.ctx, addr, cb) - s.errHandler(err) -} - -// GetBalance retrieves the balance from the given address or 0 if the account does -// not exist -func (s *VMState) GetBalance(addr common.Address) *big.Int { - res, err := s.state.GetBalance(s.ctx, addr) - s.errHandler(err) - return res -} - -// GetNonce returns the nonce at the given address or 0 if the account does -// not exist -func (s *VMState) GetNonce(addr common.Address) uint64 { - res, err := s.state.GetNonce(s.ctx, addr) - s.errHandler(err) - return res -} - -// SetNonce sets the nonce of the specified account -func (s *VMState) SetNonce(addr common.Address, nonce uint64) { - err := s.state.SetNonce(s.ctx, addr, nonce) - s.errHandler(err) -} - -// GetCode returns the contract code at the given address or nil if the account -// does not exist -func (s *VMState) GetCode(addr common.Address) []byte { - res, err := s.state.GetCode(s.ctx, addr) - s.errHandler(err) - return res -} - -// GetCodeHash returns the contract code hash at the given address -func (s *VMState) GetCodeHash(addr common.Address) common.Hash { - res, err := s.state.GetCode(s.ctx, addr) - s.errHandler(err) - return crypto.Keccak256Hash(res) -} - -// GetCodeSize returns the contract code size at the given address -func (s *VMState) GetCodeSize(addr common.Address) int { - res, err := s.state.GetCode(s.ctx, addr) - s.errHandler(err) - return len(res) -} - -// SetCode sets the contract code at the specified account -func (s *VMState) SetCode(addr common.Address, code []byte) { - err := s.state.SetCode(s.ctx, addr, code) - s.errHandler(err) -} - -// AddRefund adds an amount to the refund value collected during a vm execution -func (s *VMState) AddRefund(gas *big.Int) { - s.state.AddRefund(gas) -} - -// GetRefund returns the refund value collected during a vm execution -func (s *VMState) GetRefund() *big.Int { - return s.state.GetRefund() -} - -// GetState returns the contract storage value at storage address b from the -// contract address a or common.Hash{} if the account does not exist -func (s *VMState) GetState(a common.Address, b common.Hash) common.Hash { - res, err := s.state.GetState(s.ctx, a, b) - s.errHandler(err) - return res -} - -// SetState sets the storage value at storage address key of the account addr -func (s *VMState) SetState(addr common.Address, key common.Hash, value common.Hash) { - err := s.state.SetState(s.ctx, addr, key, value) - s.errHandler(err) -} - -// Suicide marks an account to be removed and clears its balance -func (s *VMState) Suicide(addr common.Address) bool { - res, err := s.state.Suicide(s.ctx, addr) - s.errHandler(err) - return res -} - -// Exist returns true if an account exists at the given address -func (s *VMState) Exist(addr common.Address) bool { - res, err := s.state.HasAccount(s.ctx, addr) - s.errHandler(err) - return res -} - -// Empty returns true if the account at the given address is considered empty -func (s *VMState) Empty(addr common.Address) bool { - so, err := s.state.GetStateObject(s.ctx, addr) - s.errHandler(err) - return so == nil || so.empty() -} - -// HasSuicided returns true if the given account has been marked for deletion -// or false if the account does not exist -func (s *VMState) HasSuicided(addr common.Address) bool { - res, err := s.state.HasSuicided(s.ctx, addr) - s.errHandler(err) - return res -} diff --git a/miner/worker.go b/miner/worker.go index 8030153904..e44514755a 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -274,7 +274,7 @@ func (self *worker) wait() { } go self.mux.Post(core.NewMinedBlockEvent{Block: block}) } else { - work.state.Commit(self.config.IsEIP158(block.Number())) + work.state.CommitTo(self.chainDb, self.config.IsEIP158(block.Number())) stat, err := self.chain.WriteBlock(block) if err != nil { log.Error("Failed writing block to chain", "err", err) diff --git a/tests/block_test_util.go b/tests/block_test_util.go index b9678a77bd..24d4672b64 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -204,7 +204,7 @@ func runBlockTest(homesteadBlock, daoForkBlock, gasPriceFork *big.Int, test *Blo // InsertPreState populates the given database with the genesis // accounts defined by the test. func (t *BlockTest) InsertPreState(db ethdb.Database) (*state.StateDB, error) { - statedb, err := state.New(common.Hash{}, db) + statedb, err := state.New(common.Hash{}, state.NewDatabase(db)) if err != nil { return nil, err } @@ -232,7 +232,7 @@ func (t *BlockTest) InsertPreState(db ethdb.Database) (*state.StateDB, error) { } } - root, err := statedb.Commit(false) + root, err := statedb.CommitTo(db, false) if err != nil { return nil, fmt.Errorf("error writing state: %v", err) } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index c1892cdccf..58acdd488b 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -20,7 +20,6 @@ import ( "bytes" "fmt" "io" - "math/big" "strconv" "strings" "testing" @@ -99,7 +98,7 @@ func benchStateTest(chainConfig *params.ChainConfig, test VmTest, env map[string statedb := makePreState(db, test.Pre) b.StartTimer() - RunState(chainConfig, statedb, env, test.Exec) + RunState(chainConfig, statedb, db, env, test.Exec) } func runStateTests(chainConfig *params.ChainConfig, tests map[string]VmTest, skipTests []string) error { @@ -143,16 +142,9 @@ func runStateTest(chainConfig *params.ChainConfig, test VmTest) error { env["currentTimestamp"] = test.Env.CurrentTimestamp.(string) } - var ( - ret []byte - // gas *big.Int - // err error - logs []*types.Log - ) + ret, logs, root, _ := RunState(chainConfig, statedb, db, env, test.Transaction) - ret, logs, _, _ = RunState(chainConfig, statedb, env, test.Transaction) - - // Compare expected and actual return + // Return value: var rexp []byte if strings.HasPrefix(test.Out, "#") { n, _ := strconv.Atoi(test.Out[1:]) @@ -163,61 +155,43 @@ func runStateTest(chainConfig *params.ChainConfig, test VmTest) error { if !bytes.Equal(rexp, ret) { return fmt.Errorf("return failed. Expected %x, got %x\n", rexp, ret) } - - // check post state + // Post state content: for addr, account := range test.Post { address := common.HexToAddress(addr) if !statedb.Exist(address) { return fmt.Errorf("did not find expected post-state account: %s", addr) } - if balance := statedb.GetBalance(address); balance.Cmp(math.MustParseBig256(account.Balance)) != 0 { return fmt.Errorf("(%x) balance failed. Expected: %v have: %v\n", address[:4], math.MustParseBig256(account.Balance), balance) } - if nonce := statedb.GetNonce(address); nonce != math.MustParseUint64(account.Nonce) { return fmt.Errorf("(%x) nonce failed. Expected: %v have: %v\n", address[:4], account.Nonce, nonce) } - for addr, value := range account.Storage { v := statedb.GetState(address, common.HexToHash(addr)) vexp := common.HexToHash(value) - if v != vexp { return fmt.Errorf("storage failed:\n%x: %s:\nexpected: %x\nhave: %x\n(%v %v)\n", address[:4], addr, vexp, v, vexp.Big(), v.Big()) } } } - - root, _ := statedb.Commit(false) + // Root: if common.HexToHash(test.PostStateRoot) != root { return fmt.Errorf("Post state root error. Expected: %s have: %x", test.PostStateRoot, root) } - - // check logs - if len(test.Logs) > 0 { - if err := checkLogs(test.Logs, logs); err != nil { - return err - } - } - - return nil + // Logs: + return checkLogs(test.Logs, logs) } -func RunState(chainConfig *params.ChainConfig, statedb *state.StateDB, env, tx map[string]string) ([]byte, []*types.Log, *big.Int, error) { +func RunState(chainConfig *params.ChainConfig, statedb *state.StateDB, db ethdb.Database, env, tx map[string]string) ([]byte, []*types.Log, common.Hash, error) { environment, msg := NewEVMEnvironment(false, chainConfig, statedb, env, tx) gaspool := new(core.GasPool).AddGas(math.MustParseBig256(env["currentGasLimit"])) - root, _ := statedb.Commit(false) - statedb.Reset(root) - snapshot := statedb.Snapshot() - - ret, gasUsed, err := core.ApplyMessage(environment, msg, gaspool) + ret, _, err := core.ApplyMessage(environment, msg, gaspool) if err != nil { statedb.RevertToSnapshot(snapshot) } - statedb.Commit(chainConfig.IsEIP158(environment.Context.BlockNumber)) - - return ret, statedb.Logs(), gasUsed, err + root, _ := statedb.CommitTo(db, chainConfig.IsEIP158(environment.Context.BlockNumber)) + return ret, statedb.Logs(), root, err } diff --git a/tests/util.go b/tests/util.go index a3a9a1f645..ff02679ec4 100644 --- a/tests/util.go +++ b/tests/util.go @@ -48,7 +48,6 @@ func init() { } func checkLogs(tlog []Log, logs []*types.Log) error { - if len(tlog) != len(logs) { return fmt.Errorf("log length mismatch. Expected %d, got %d", len(tlog), len(logs)) } else { @@ -106,10 +105,14 @@ func (self Log) Topics() [][]byte { } func makePreState(db ethdb.Database, accounts map[string]Account) *state.StateDB { - statedb, _ := state.New(common.Hash{}, db) + sdb := state.NewDatabase(db) + statedb, _ := state.New(common.Hash{}, sdb) for addr, account := range accounts { insertAccount(statedb, addr, account) } + // Commit and re-open to start with a clean state. + root, _ := statedb.CommitTo(db, false) + statedb, _ = state.New(root, sdb) return statedb } diff --git a/trie/proof.go b/trie/proof.go index 1f8f76b1b7..298f648c4b 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -125,7 +125,7 @@ func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value } func get(tn node, key []byte) ([]byte, node) { - for len(key) > 0 { + for { switch n := tn.(type) { case *shortNode: if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) { @@ -140,9 +140,10 @@ func get(tn node, key []byte) ([]byte, node) { return key, n case nil: return key, nil + case valueNode: + return nil, n default: panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) } } - return nil, tn.(valueNode) } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 37d1d4b092..20c303f31c 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -156,6 +156,11 @@ func (t *SecureTrie) Root() []byte { return t.trie.Root() } +func (t *SecureTrie) Copy() *SecureTrie { + cpy := *t + return &cpy +} + // NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration // starts at the key after the given start key. func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { From bae7565231376bbd23474e2e91c99a21542ef47a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 28 Jun 2017 09:51:31 +0200 Subject: [PATCH 24/43] core/vm: fix overflow in gas calculation formula --- core/vm/gas_table.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 24ad6caa52..761ca44507 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -17,7 +17,6 @@ package vm import ( - gmath "math" "math/big" "github.com/ethereum/go-ethereum/common" @@ -28,15 +27,20 @@ import ( // memoryGasCosts calculates the quadratic gas for memory expansion. It does so // only for the memory region that is expanded, not the total memory. func memoryGasCost(mem *Memory, newMemSize uint64) (uint64, error) { - // The maximum that will fit in a uint64 is max_word_count - 1 - // anything above that will result in an overflow. - if newMemSize > gmath.MaxUint64-32 { - return 0, errGasUintOverflow - } if newMemSize == 0 { return 0, nil } + // The maximum that will fit in a uint64 is max_word_count - 1 + // anything above that will result in an overflow. + // Additionally, a newMemSize which results in a + // newMemSizeWords larger than 0x7ffffffff will cause the square operation + // to overflow. + // The constant 0xffffffffe0 is the highest number that can be used without + // overflowing the gas calculation + if newMemSize > 0xffffffffe0 { + return 0, errGasUintOverflow + } newMemSizeWords := toWordSize(newMemSize) newMemSize = newMemSizeWords * 32 From e4301564c2af0b4f084a3bcf8ddcec5e84ac4ead Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 28 Jun 2017 10:47:07 +0200 Subject: [PATCH 25/43] core/vm : fix testcase for gas calculation --- core/vm/gas_table_test.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 1ee909e92b..1b91aee56c 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -16,24 +16,20 @@ package vm -import ( - "math" - "testing" -) +import "testing" func TestMemoryGasCost(t *testing.T) { - size := uint64(math.MaxUint64 - 64) - _, err := memoryGasCost(&Memory{}, size) + //size := uint64(math.MaxUint64 - 64) + size := uint64(0xffffffffe0) + v, err := memoryGasCost(&Memory{}, size) if err != nil { t.Error("didn't expect error:", err) } - - _, err = memoryGasCost(&Memory{}, size+32) - if err != nil { - t.Error("didn't expect error:", err) + if v != 36028899963961341 { + t.Errorf("Expected: 36028899963961341, got %d", v) } - _, err = memoryGasCost(&Memory{}, size+33) + _, err = memoryGasCost(&Memory{}, size+1) if err == nil { t.Error("expected error") } From 6dc32e897a0e71982adbbf7123e19115f27f7135 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 28 Jun 2017 11:45:45 +0200 Subject: [PATCH 26/43] core/vm: add benchmarks for some ops and precompiles (#14641) --- core/vm/instructions_test.go | 242 +++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index ae428aeaba..03c42c561c 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -1,6 +1,7 @@ package vm import ( + "fmt" "math/big" "testing" @@ -40,3 +41,244 @@ func TestByteOp(t *testing.T) { } } } + +func opBenchmark(bench *testing.B, op func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error), args ...string) { + var ( + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) + stack = newstack() + ) + // convert args + byteArgs := make([][]byte, len(args)) + for i, arg := range args { + byteArgs[i] = common.Hex2Bytes(arg) + } + pc := uint64(0) + bench.ResetTimer() + for i := 0; i < bench.N; i++ { + for _, arg := range byteArgs { + a := new(big.Int).SetBytes(arg) + stack.push(a) + } + op(&pc, env, nil, nil, stack) + stack.pop() + } +} + +func precompiledBenchmark(addr, input, expected string, gas uint64, bench *testing.B) { + + contract := NewContract(AccountRef(common.HexToAddress("1337")), + nil, new(big.Int), gas) + + p := PrecompiledContracts[common.HexToAddress(addr)] + in := common.Hex2Bytes(input) + var ( + res []byte + err error + ) + data := make([]byte, len(in)) + bench.ResetTimer() + for i := 0; i < bench.N; i++ { + contract.Gas = gas + copy(data, in) + res, err = RunPrecompiledContract(p, data, contract) + } + bench.StopTimer() + //Check if it is correct + if err != nil { + bench.Error(err) + return + } + if common.Bytes2Hex(res) != expected { + bench.Error(fmt.Sprintf("Expected %v, got %v", expected, common.Bytes2Hex(res))) + return + } +} + +func BenchmarkPrecompiledEcdsa(bench *testing.B) { + var ( + addr = "01" + inp = "38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e000000000000000000000000000000000000000000000000000000000000001b38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e789d1dd423d25f0772d2748d60f7e4b81bb14d086eba8e8e8efb6dcff8a4ae02" + exp = "000000000000000000000000ceaccac640adf55b2028469bd36ba501f28b699d" + gas = uint64(4000000) + ) + precompiledBenchmark(addr, inp, exp, gas, bench) +} +func BenchmarkPrecompiledSha256(bench *testing.B) { + var ( + addr = "02" + inp = "38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e000000000000000000000000000000000000000000000000000000000000001b38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e789d1dd423d25f0772d2748d60f7e4b81bb14d086eba8e8e8efb6dcff8a4ae02" + exp = "811c7003375852fabd0d362e40e68607a12bdabae61a7d068fe5fdd1dbbf2a5d" + gas = uint64(4000000) + ) + precompiledBenchmark(addr, inp, exp, gas, bench) +} +func BenchmarkPrecompiledRipeMD(bench *testing.B) { + var ( + addr = "03" + inp = "38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e000000000000000000000000000000000000000000000000000000000000001b38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e789d1dd423d25f0772d2748d60f7e4b81bb14d086eba8e8e8efb6dcff8a4ae02" + exp = "0000000000000000000000009215b8d9882ff46f0dfde6684d78e831467f65e6" + gas = uint64(4000000) + ) + precompiledBenchmark(addr, inp, exp, gas, bench) +} +func BenchmarkPrecompiledIdentity(bench *testing.B) { + var ( + addr = "04" + inp = "38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e000000000000000000000000000000000000000000000000000000000000001b38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e789d1dd423d25f0772d2748d60f7e4b81bb14d086eba8e8e8efb6dcff8a4ae02" + exp = "38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e000000000000000000000000000000000000000000000000000000000000001b38d18acb67d25c8bb9942764b62f18e17054f66a817bd4295423adf9ed98873e789d1dd423d25f0772d2748d60f7e4b81bb14d086eba8e8e8efb6dcff8a4ae02" + gas = uint64(4000000) + ) + precompiledBenchmark(addr, inp, exp, gas, bench) +} +func BenchmarkOpAdd(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opAdd, x, y) + +} +func BenchmarkOpSub(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opSub, x, y) + +} +func BenchmarkOpMul(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opMul, x, y) + +} +func BenchmarkOpDiv(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opDiv, x, y) + +} +func BenchmarkOpSdiv(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opSdiv, x, y) + +} +func BenchmarkOpMod(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opMod, x, y) + +} +func BenchmarkOpSmod(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opSmod, x, y) + +} +func BenchmarkOpExp(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opExp, x, y) + +} +func BenchmarkOpSignExtend(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opSignExtend, x, y) + +} +func BenchmarkOpLt(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opLt, x, y) + +} +func BenchmarkOpGt(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opGt, x, y) + +} +func BenchmarkOpSlt(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opSlt, x, y) + +} +func BenchmarkOpSgt(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opSgt, x, y) + +} +func BenchmarkOpEq(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opEq, x, y) + +} +func BenchmarkOpAnd(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opAnd, x, y) + +} +func BenchmarkOpOr(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opOr, x, y) + +} +func BenchmarkOpXor(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opXor, x, y) + +} +func BenchmarkOpByte(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opByte, x, y) + +} + +func BenchmarkOpAddmod(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + z := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opAddmod, x, y, z) + +} +func BenchmarkOpMulmod(b *testing.B) { + x := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + y := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + z := "ABCDEF090807060504030201ffffffffffffffffffffffffffffffffffffffff" + + opBenchmark(b, opMulmod, x, y, z) + +} + +//func BenchmarkOpSha3(b *testing.B) { +// x := "0" +// y := "32" +// +// opBenchmark(b,opSha3, x, y) +// +// +//} From 055095798911e3c6f2e210220cb7ece1b71f6517 Mon Sep 17 00:00:00 2001 From: Nick Johnson Date: Tue, 27 Jun 2017 16:15:29 +0100 Subject: [PATCH 27/43] eth, les, light: Refactor downloader to use blockchain interface --- eth/downloader/downloader.go | 143 +++++++++++++++++------------- eth/downloader/downloader_test.go | 72 ++++++++------- eth/downloader/types.go | 41 --------- eth/handler.go | 5 +- les/handler.go | 4 +- light/lightchain.go | 15 ++++ 6 files changed, 134 insertions(+), 146 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index e4d1392d0a..a5d03d17e8 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -114,21 +114,11 @@ type Downloader struct { syncStatsState stateSyncStats syncStatsLock sync.RWMutex // Lock protecting the sync stats fields + lightchain LightChain + chain BlockChain + // Callbacks - hasHeader headerCheckFn // Checks if a header is present in the chain - hasBlockAndState blockAndStateCheckFn // Checks if a block and associated state is present in the chain - getHeader headerRetrievalFn // Retrieves a header from the chain - getBlock blockRetrievalFn // Retrieves a block from the chain - headHeader headHeaderRetrievalFn // Retrieves the head header from the chain - headBlock headBlockRetrievalFn // Retrieves the head block from the chain - headFastBlock headFastBlockRetrievalFn // Retrieves the head fast-sync block from the chain - commitHeadBlock headBlockCommitterFn // Commits a manually assembled block as the chain head - getTd tdRetrievalFn // Retrieves the TD of a block from the chain - insertHeaders headerChainInsertFn // Injects a batch of headers into the chain - insertBlocks blockChainInsertFn // Injects a batch of blocks into the chain - insertReceipts receiptChainInsertFn // Injects a batch of blocks and their receipts into the chain - rollback chainRollbackFn // Removes a batch of recently added chain links - dropPeer peerDropFn // Drops a peer for misbehaving + dropPeer peerDropFn // Drops a peer for misbehaving // Status synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing @@ -163,11 +153,56 @@ type Downloader struct { chainInsertHook func([]*fetchResult) // Method to call upon inserting a chain of blocks (possibly in multiple invocations) } +type LightChain interface { + // HasHeader verifies a header's presence in the local chain. + HasHeader(common.Hash) bool + + // GetHeaderByHash retrieves a header from the local chain. + GetHeaderByHash(common.Hash) *types.Header + + // CurrentHeader retrieves the head header from the local chain. + CurrentHeader() *types.Header + + // GetTdByHash returns the total difficulty of a local block. + GetTdByHash(common.Hash) *big.Int + + // InsertHeaderChain inserts a batch of headers into the local chain. + InsertHeaderChain([]*types.Header, int) (int, error) + + // Rollback removes a few recently added elements from the local chain. + Rollback([]common.Hash) +} + +type BlockChain interface { + LightChain + + // HasBlockAndState verifies block and associated states' presence in the local chain. + HasBlockAndState(common.Hash) bool + + // GetBlockByHash retrieves a block from the local chain. + GetBlockByHash(common.Hash) *types.Block + + // CurrentBlock retrieves the head block from the local chain. + CurrentBlock() *types.Block + + // CurrentFastBlock retrieves the head fast block from the local chain. + CurrentFastBlock() *types.Block + + // FastSyncCommitHead directly commits the head block to a certain entity. + FastSyncCommitHead(common.Hash) error + + // InsertChain inserts a batch of blocks into the local chain. + InsertChain(types.Blocks) (int, error) + + // InsertReceiptChain inserts a batch of receipts into the local chain. + InsertReceiptChain(types.Blocks, []types.Receipts) (int, error) +} + // New creates a new downloader to fetch hashes and blocks from remote peers. -func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, hasBlockAndState blockAndStateCheckFn, - getHeader headerRetrievalFn, getBlock blockRetrievalFn, headHeader headHeaderRetrievalFn, headBlock headBlockRetrievalFn, - headFastBlock headFastBlockRetrievalFn, commitHeadBlock headBlockCommitterFn, getTd tdRetrievalFn, insertHeaders headerChainInsertFn, - insertBlocks blockChainInsertFn, insertReceipts receiptChainInsertFn, rollback chainRollbackFn, dropPeer peerDropFn) *Downloader { +func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, chain BlockChain, lightchain LightChain, dropPeer peerDropFn) *Downloader { + if lightchain == nil { + lightchain = chain + } dl := &Downloader{ mode: mode, @@ -177,19 +212,8 @@ func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, hasHeader he stateDB: stateDb, rttEstimate: uint64(rttMaxEstimate), rttConfidence: uint64(1000000), - hasHeader: hasHeader, - hasBlockAndState: hasBlockAndState, - getHeader: getHeader, - getBlock: getBlock, - headHeader: headHeader, - headBlock: headBlock, - headFastBlock: headFastBlock, - commitHeadBlock: commitHeadBlock, - getTd: getTd, - insertHeaders: insertHeaders, - insertBlocks: insertBlocks, - insertReceipts: insertReceipts, - rollback: rollback, + chain: chain, + lightchain: lightchain, dropPeer: dropPeer, headerCh: make(chan dataPack, 1), bodyCh: make(chan dataPack, 1), @@ -223,11 +247,11 @@ func (d *Downloader) Progress() ethereum.SyncProgress { current := uint64(0) switch d.mode { case FullSync: - current = d.headBlock().NumberU64() + current = d.chain.CurrentBlock().NumberU64() case FastSync: - current = d.headFastBlock().NumberU64() + current = d.chain.CurrentFastBlock().NumberU64() case LightSync: - current = d.headHeader().Number.Uint64() + current = d.lightchain.CurrentHeader().Number.Uint64() } return ethereum.SyncProgress{ StartingBlock: d.syncStatsChainOrigin, @@ -572,13 +596,13 @@ func (d *Downloader) fetchHeight(p *peer) (*types.Header, error) { // the head links match), we do a binary search to find the common ancestor. func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { // Figure out the valid ancestor range to prevent rewrite attacks - floor, ceil := int64(-1), d.headHeader().Number.Uint64() + floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64() p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height) if d.mode == FullSync { - ceil = d.headBlock().NumberU64() + ceil = d.chain.CurrentBlock().NumberU64() } else if d.mode == FastSync { - ceil = d.headFastBlock().NumberU64() + ceil = d.chain.CurrentFastBlock().NumberU64() } if ceil >= MaxForkAncestry { floor = int64(ceil - MaxForkAncestry) @@ -638,7 +662,7 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { continue } // Otherwise check if we already know the header or not - if (d.mode == FullSync && d.hasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.hasHeader(headers[i].Hash())) { + if (d.mode == FullSync && d.chain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash())) { number, hash = headers[i].Number.Uint64(), headers[i].Hash() // If every header is known, even future ones, the peer straight out lied about its head @@ -703,11 +727,11 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { arrived = true // Modify the search interval based on the response - if (d.mode == FullSync && !d.hasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.hasHeader(headers[0].Hash())) { + if (d.mode == FullSync && !d.chain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash())) { end = check break } - header := d.getHeader(headers[0].Hash()) // Independent of sync mode, header surely exists + header := d.lightchain.GetHeaderByHash(headers[0].Hash()) // Independent of sync mode, header surely exists if header.Number.Uint64() != check { p.log.Debug("Received non requested header", "number", header.Number, "hash", header.Hash(), "request", check) return 0, errBadPeer @@ -1124,23 +1148,19 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { for i, header := range rollback { hashes[i] = header.Hash() } - lastHeader, lastFastBlock, lastBlock := d.headHeader().Number, common.Big0, common.Big0 - if d.headFastBlock != nil { - lastFastBlock = d.headFastBlock().Number() + lastHeader, lastFastBlock, lastBlock := d.lightchain.CurrentHeader().Number, common.Big0, common.Big0 + if d.mode != LightSync { + lastFastBlock = d.chain.CurrentFastBlock().Number() + lastBlock = d.chain.CurrentBlock().Number() } - if d.headBlock != nil { - lastBlock = d.headBlock().Number() - } - d.rollback(hashes) + d.lightchain.Rollback(hashes) curFastBlock, curBlock := common.Big0, common.Big0 - if d.headFastBlock != nil { - curFastBlock = d.headFastBlock().Number() - } - if d.headBlock != nil { - curBlock = d.headBlock().Number() + if d.mode != LightSync { + curFastBlock = d.chain.CurrentFastBlock().Number() + curBlock = d.chain.CurrentBlock().Number() } log.Warn("Rolled back headers", "count", len(hashes), - "header", fmt.Sprintf("%d->%d", lastHeader, d.headHeader().Number), + "header", fmt.Sprintf("%d->%d", lastHeader, d.lightchain.CurrentHeader().Number), "fast", fmt.Sprintf("%d->%d", lastFastBlock, curFastBlock), "block", fmt.Sprintf("%d->%d", lastBlock, curBlock)) @@ -1190,7 +1210,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { // L: Request new headers up from 11 (R's TD was higher, it must have something) // R: Nothing to give if d.mode != LightSync { - if !gotHeaders && td.Cmp(d.getTd(d.headBlock().Hash())) > 0 { + if !gotHeaders && td.Cmp(d.chain.GetTdByHash(d.chain.CurrentBlock().Hash())) > 0 { return errStallingPeer } } @@ -1202,7 +1222,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { // queued for processing when the header download completes. However, as long as the // peer gave us something useful, we're already happy/progressed (above check). if d.mode == FastSync || d.mode == LightSync { - if td.Cmp(d.getTd(d.headHeader().Hash())) > 0 { + if td.Cmp(d.lightchain.GetTdByHash(d.lightchain.CurrentHeader().Hash())) > 0 { return errStallingPeer } } @@ -1232,7 +1252,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { // Collect the yet unknown headers to mark them as uncertain unknown := make([]*types.Header, 0, len(headers)) for _, header := range chunk { - if !d.hasHeader(header.Hash()) { + if !d.lightchain.HasHeader(header.Hash()) { unknown = append(unknown, header) } } @@ -1241,7 +1261,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { if chunk[len(chunk)-1].Number.Uint64()+uint64(fsHeaderForceVerify) > pivot { frequency = 1 } - if n, err := d.insertHeaders(chunk, frequency); err != nil { + if n, err := d.chain.InsertHeaderChain(chunk, frequency); err != nil { // If some headers were inserted, add them too to the rollback list if n > 0 { rollback = append(rollback, chunk[:n]...) @@ -1328,7 +1348,7 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { for i, result := range results[:items] { blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) } - if index, err := d.insertBlocks(blocks); err != nil { + if index, err := d.chain.InsertChain(blocks); err != nil { log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) return errInvalidChain } @@ -1368,6 +1388,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error { stateSync.Cancel() if err := d.commitPivotBlock(P); err != nil { return err + } } if err := d.importBlockResults(afterP); err != nil { @@ -1416,7 +1437,7 @@ func (d *Downloader) commitFastSyncData(results []*fetchResult, stateSync *state blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) receipts[i] = result.Receipts } - if index, err := d.insertReceipts(blocks, receipts); err != nil { + if index, err := d.chain.InsertReceiptChain(blocks, receipts); err != nil { log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) return errInvalidChain } @@ -1434,10 +1455,10 @@ func (d *Downloader) commitPivotBlock(result *fetchResult) error { return err } log.Debug("Committing fast sync pivot as new head", "number", b.Number(), "hash", b.Hash()) - if _, err := d.insertReceipts([]*types.Block{b}, []types.Receipts{result.Receipts}); err != nil { + if _, err := d.chain.InsertReceiptChain([]*types.Block{b}, []types.Receipts{result.Receipts}); err != nil { return err } - return d.commitHeadBlock(b.Hash()) + return d.chain.FastSyncCommitHead(b.Hash()) } // DeliverHeaders injects a new batch of block headers received from a remote diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 1fb5a0910e..5852127ea7 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -96,9 +96,7 @@ func newTester() *downloadTester { tester.stateDb, _ = ethdb.NewMemDatabase() tester.stateDb.Put(genesis.Root().Bytes(), []byte{0x00}) - tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester.hasHeader, tester.hasBlock, tester.getHeader, - tester.getBlock, tester.headHeader, tester.headBlock, tester.headFastBlock, tester.commitHeadBlock, tester.getTd, - tester.insertHeaders, tester.insertBlocks, tester.insertReceipts, tester.rollback, tester.dropPeer) + tester.downloader = New(FullSync, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) return tester } @@ -218,14 +216,14 @@ func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { return err } -// hasHeader checks if a header is present in the testers canonical chain. -func (dl *downloadTester) hasHeader(hash common.Hash) bool { - return dl.getHeader(hash) != nil +// HasHeader checks if a header is present in the testers canonical chain. +func (dl *downloadTester) HasHeader(hash common.Hash) bool { + return dl.GetHeaderByHash(hash) != nil } -// hasBlock checks if a block and associated state is present in the testers canonical chain. -func (dl *downloadTester) hasBlock(hash common.Hash) bool { - block := dl.getBlock(hash) +// HasBlockAndState checks if a block and associated state is present in the testers canonical chain. +func (dl *downloadTester) HasBlockAndState(hash common.Hash) bool { + block := dl.GetBlockByHash(hash) if block == nil { return false } @@ -233,24 +231,24 @@ func (dl *downloadTester) hasBlock(hash common.Hash) bool { return err == nil } -// getHeader retrieves a header from the testers canonical chain. -func (dl *downloadTester) getHeader(hash common.Hash) *types.Header { +// GetHeader retrieves a header from the testers canonical chain. +func (dl *downloadTester) GetHeaderByHash(hash common.Hash) *types.Header { dl.lock.RLock() defer dl.lock.RUnlock() return dl.ownHeaders[hash] } -// getBlock retrieves a block from the testers canonical chain. -func (dl *downloadTester) getBlock(hash common.Hash) *types.Block { +// GetBlock retrieves a block from the testers canonical chain. +func (dl *downloadTester) GetBlockByHash(hash common.Hash) *types.Block { dl.lock.RLock() defer dl.lock.RUnlock() return dl.ownBlocks[hash] } -// headHeader retrieves the current head header from the canonical chain. -func (dl *downloadTester) headHeader() *types.Header { +// CurrentHeader retrieves the current head header from the canonical chain. +func (dl *downloadTester) CurrentHeader() *types.Header { dl.lock.RLock() defer dl.lock.RUnlock() @@ -262,8 +260,8 @@ func (dl *downloadTester) headHeader() *types.Header { return dl.genesis.Header() } -// headBlock retrieves the current head block from the canonical chain. -func (dl *downloadTester) headBlock() *types.Block { +// CurrentBlock retrieves the current head block from the canonical chain. +func (dl *downloadTester) CurrentBlock() *types.Block { dl.lock.RLock() defer dl.lock.RUnlock() @@ -277,8 +275,8 @@ func (dl *downloadTester) headBlock() *types.Block { return dl.genesis } -// headFastBlock retrieves the current head fast-sync block from the canonical chain. -func (dl *downloadTester) headFastBlock() *types.Block { +// CurrentFastBlock retrieves the current head fast-sync block from the canonical chain. +func (dl *downloadTester) CurrentFastBlock() *types.Block { dl.lock.RLock() defer dl.lock.RUnlock() @@ -290,26 +288,26 @@ func (dl *downloadTester) headFastBlock() *types.Block { return dl.genesis } -// commitHeadBlock manually sets the head block to a given hash. -func (dl *downloadTester) commitHeadBlock(hash common.Hash) error { +// FastSynccommitHead manually sets the head block to a given hash. +func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct - if block := dl.getBlock(hash); block != nil { + if block := dl.GetBlockByHash(hash); block != nil { _, err := trie.NewSecure(block.Root(), dl.stateDb, 0) return err } return fmt.Errorf("non existent block: %x", hash[:4]) } -// getTd retrieves the block's total difficulty from the canonical chain. -func (dl *downloadTester) getTd(hash common.Hash) *big.Int { +// GetTdByHash retrieves the block's total difficulty from the canonical chain. +func (dl *downloadTester) GetTdByHash(hash common.Hash) *big.Int { dl.lock.RLock() defer dl.lock.RUnlock() return dl.ownChainTd[hash] } -// insertHeaders injects a new batch of headers into the simulated chain. -func (dl *downloadTester) insertHeaders(headers []*types.Header, checkFreq int) (int, error) { +// InsertHeaderChain injects a new batch of headers into the simulated chain. +func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (int, error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -337,8 +335,8 @@ func (dl *downloadTester) insertHeaders(headers []*types.Header, checkFreq int) return len(headers), nil } -// insertBlocks injects a new batch of blocks into the simulated chain. -func (dl *downloadTester) insertBlocks(blocks types.Blocks) (int, error) { +// InsertChain injects a new batch of blocks into the simulated chain. +func (dl *downloadTester) InsertChain(blocks types.Blocks) (int, error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -359,8 +357,8 @@ func (dl *downloadTester) insertBlocks(blocks types.Blocks) (int, error) { return len(blocks), nil } -// insertReceipts injects a new batch of receipts into the simulated chain. -func (dl *downloadTester) insertReceipts(blocks types.Blocks, receipts []types.Receipts) (int, error) { +// InsertReceiptChain injects a new batch of receipts into the simulated chain. +func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts) (int, error) { dl.lock.Lock() defer dl.lock.Unlock() @@ -377,8 +375,8 @@ func (dl *downloadTester) insertReceipts(blocks types.Blocks, receipts []types.R return len(blocks), nil } -// rollback removes some recently added elements from the chain. -func (dl *downloadTester) rollback(hashes []common.Hash) { +// Rollback removes some recently added elements from the chain. +func (dl *downloadTester) Rollback(hashes []common.Hash) { dl.lock.Lock() defer dl.lock.Unlock() @@ -1212,7 +1210,7 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { if err := tester.sync("fast-attack", nil, mode); err == nil { t.Fatalf("succeeded fast attacker synchronisation") } - if head := tester.headHeader().Number.Int64(); int(head) > MaxHeaderFetch { + if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch) } // Attempt to sync with an attacker that feeds junk during the block import phase. @@ -1226,11 +1224,11 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { if err := tester.sync("block-attack", nil, mode); err == nil { t.Fatalf("succeeded block attacker synchronisation") } - if head := tester.headHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { + if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch) } if mode == FastSync { - if head := tester.headBlock().NumberU64(); head != 0 { + if head := tester.CurrentBlock().NumberU64(); head != 0 { t.Errorf("fast sync pivot block #%d not rolled back", head) } } @@ -1251,11 +1249,11 @@ func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) { if err := tester.sync("withhold-attack", nil, mode); err == nil { t.Fatalf("succeeded withholding attacker synchronisation") } - if head := tester.headHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { + if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch) } if mode == FastSync { - if head := tester.headBlock().NumberU64(); head != 0 { + if head := tester.CurrentBlock().NumberU64(); head != 0 { t.Errorf("fast sync pivot block #%d not rolled back", head) } } diff --git a/eth/downloader/types.go b/eth/downloader/types.go index e105104864..3f30ea9dd1 100644 --- a/eth/downloader/types.go +++ b/eth/downloader/types.go @@ -18,51 +18,10 @@ package downloader import ( "fmt" - "math/big" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" ) -// headerCheckFn is a callback type for verifying a header's presence in the local chain. -type headerCheckFn func(common.Hash) bool - -// blockAndStateCheckFn is a callback type for verifying block and associated states' presence in the local chain. -type blockAndStateCheckFn func(common.Hash) bool - -// headerRetrievalFn is a callback type for retrieving a header from the local chain. -type headerRetrievalFn func(common.Hash) *types.Header - -// blockRetrievalFn is a callback type for retrieving a block from the local chain. -type blockRetrievalFn func(common.Hash) *types.Block - -// headHeaderRetrievalFn is a callback type for retrieving the head header from the local chain. -type headHeaderRetrievalFn func() *types.Header - -// headBlockRetrievalFn is a callback type for retrieving the head block from the local chain. -type headBlockRetrievalFn func() *types.Block - -// headFastBlockRetrievalFn is a callback type for retrieving the head fast block from the local chain. -type headFastBlockRetrievalFn func() *types.Block - -// headBlockCommitterFn is a callback for directly committing the head block to a certain entity. -type headBlockCommitterFn func(common.Hash) error - -// tdRetrievalFn is a callback type for retrieving the total difficulty of a local block. -type tdRetrievalFn func(common.Hash) *big.Int - -// headerChainInsertFn is a callback type to insert a batch of headers into the local chain. -type headerChainInsertFn func([]*types.Header, int) (int, error) - -// blockChainInsertFn is a callback type to insert a batch of blocks into the local chain. -type blockChainInsertFn func(types.Blocks) (int, error) - -// receiptChainInsertFn is a callback type to insert a batch of receipts into the local chain. -type receiptChainInsertFn func(types.Blocks, []types.Receipts) (int, error) - -// chainRollbackFn is a callback type to remove a few recently added elements from the local chain. -type chainRollbackFn func([]common.Hash) - // peerDropFn is a callback type for dropping a peer detected as malicious. type peerDropFn func(id string) diff --git a/eth/handler.go b/eth/handler.go index 1af9e755ba..8d026c7757 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -157,10 +157,7 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne return nil, errIncompatibleConfig } // Construct the different synchronisation mechanisms - manager.downloader = downloader.New(mode, chaindb, manager.eventMux, blockchain.HasHeader, blockchain.HasBlockAndState, blockchain.GetHeaderByHash, - blockchain.GetBlockByHash, blockchain.CurrentHeader, blockchain.CurrentBlock, blockchain.CurrentFastBlock, blockchain.FastSyncCommitHead, - blockchain.GetTdByHash, blockchain.InsertHeaderChain, manager.blockchain.InsertChain, blockchain.InsertReceiptChain, blockchain.Rollback, - manager.removePeer) + manager.downloader = downloader.New(mode, chaindb, manager.eventMux, blockchain, nil, manager.removePeer) validator := func(header *types.Header) error { return engine.VerifyHeader(blockchain, header, true) diff --git a/les/handler.go b/les/handler.go index 77bc077a2e..afd4356efe 100644 --- a/les/handler.go +++ b/les/handler.go @@ -206,9 +206,7 @@ func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, network } if lightSync { - manager.downloader = downloader.New(downloader.LightSync, chainDb, manager.eventMux, blockchain.HasHeader, nil, blockchain.GetHeaderByHash, - nil, blockchain.CurrentHeader, nil, nil, nil, blockchain.GetTdByHash, - blockchain.InsertHeaderChain, nil, nil, blockchain.Rollback, removePeer) + manager.downloader = downloader.New(downloader.LightSync, chainDb, manager.eventMux, nil, blockchain, removePeer) manager.peers.notify((*downloaderPeerNotify)(manager)) manager.fetcher = newLightFetcher(manager) } diff --git a/light/lightchain.go b/light/lightchain.go index 87436f4a57..ec7ca3a0c2 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -389,6 +389,21 @@ func (self *LightChain) CurrentHeader() *types.Header { return self.hc.CurrentHeader() } +// CurrentBlock exists for interface compatibility and always returns nil +func (self *LightChain) CurrentBlock() *types.Block { + return nil +} + +// CurrentFastBlock exists for interface compatibility and always returns nil +func (self *LightChain) CurrentFastBlock() *types.Block { + return nil +} + +// FastSyncCommitHead exists for interface compatibility and does nothing +func (self *LightChain) FastSyncCommitHead(h common.Hash) error { + return nil +} + // GetTd retrieves a block's total difficulty in the canonical chain from the // database by hash and number, caching it if found. func (self *LightChain) GetTd(hash common.Hash, number uint64) *big.Int { From ae11545bc5fc0fcf4340d9f33d488eaf81d8bb2d Mon Sep 17 00:00:00 2001 From: Nick Johnson Date: Wed, 28 Jun 2017 13:25:08 +0100 Subject: [PATCH 28/43] eth, les: Refactor downloader peer to use structs --- eth/downloader/downloader.go | 91 ++++++----- eth/downloader/downloader_test.go | 254 ++++++++++++++++-------------- eth/downloader/peer.go | 130 +++++++-------- eth/downloader/queue.go | 10 +- eth/downloader/statesync.go | 4 +- eth/handler.go | 2 +- les/handler.go | 122 ++++++++------ 7 files changed, 316 insertions(+), 297 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index a5d03d17e8..0861e917a1 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -205,27 +205,26 @@ func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, chain BlockC } dl := &Downloader{ - mode: mode, - mux: mux, - queue: newQueue(), - peers: newPeerSet(), - stateDB: stateDb, - rttEstimate: uint64(rttMaxEstimate), - rttConfidence: uint64(1000000), - chain: chain, - lightchain: lightchain, - dropPeer: dropPeer, - headerCh: make(chan dataPack, 1), - bodyCh: make(chan dataPack, 1), - receiptCh: make(chan dataPack, 1), - bodyWakeCh: make(chan bool, 1), - receiptWakeCh: make(chan bool, 1), - headerProcCh: make(chan []*types.Header, 1), - quitCh: make(chan struct{}), - // for stateFetcher + mode: mode, + stateDB: stateDb, + mux: mux, + queue: newQueue(), + peers: newPeerSet(), + rttEstimate: uint64(rttMaxEstimate), + rttConfidence: uint64(1000000), + chain: chain, + lightchain: lightchain, + dropPeer: dropPeer, + headerCh: make(chan dataPack, 1), + bodyCh: make(chan dataPack, 1), + receiptCh: make(chan dataPack, 1), + bodyWakeCh: make(chan bool, 1), + receiptWakeCh: make(chan bool, 1), + headerProcCh: make(chan []*types.Header, 1), + quitCh: make(chan struct{}), + stateCh: make(chan dataPack), stateSyncStart: make(chan *stateSync), trackStateReq: make(chan *stateReq), - stateCh: make(chan dataPack), } go dl.qosTuner() go dl.stateFetcher() @@ -269,13 +268,11 @@ func (d *Downloader) Synchronising() bool { // RegisterPeer injects a new download peer into the set of block source to be // used for fetching hashes and blocks from. -func (d *Downloader) RegisterPeer(id string, version int, currentHead currentHeadRetrievalFn, - getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn, - getReceipts receiptFetcherFn, getNodeData stateFetcherFn) error { +func (d *Downloader) RegisterPeer(id string, version int, peer Peer) error { logger := log.New("peer", id) logger.Trace("Registering sync peer") - if err := d.peers.Register(newPeer(id, version, currentHead, getRelHeaders, getAbsHeaders, getBlockBodies, getReceipts, getNodeData, logger)); err != nil { + if err := d.peers.Register(newPeerConnection(id, version, peer, logger)); err != nil { logger.Error("Failed to register sync peer", "err", err) return err } @@ -395,7 +392,7 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode // syncWithPeer starts a block synchronization based on the hash chain from the // specified peer and head hash. -func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err error) { +func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.Int) (err error) { d.mux.Post(StartEvent{}) defer func() { // reset on error @@ -548,12 +545,12 @@ func (d *Downloader) Terminate() { // fetchHeight retrieves the head header of the remote peer to aid in estimating // the total time a pending synchronisation would take. -func (d *Downloader) fetchHeight(p *peer) (*types.Header, error) { +func (d *Downloader) fetchHeight(p *peerConnection) (*types.Header, error) { p.log.Debug("Retrieving remote chain height") // Request the advertised remote head block and wait for the response - head, _ := p.currentHead() - go p.getRelHeaders(head, 1, 0, false) + head, _ := p.peer.Head() + go p.peer.RequestHeadersByHash(head, 1, 0, false) ttl := d.requestTTL() timeout := time.After(ttl) @@ -594,7 +591,7 @@ func (d *Downloader) fetchHeight(p *peer) (*types.Header, error) { // on the correct chain, checking the top N links should already get us a match. // In the rare scenario when we ended up on a long reorganisation (i.e. none of // the head links match), we do a binary search to find the common ancestor. -func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { +func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, error) { // Figure out the valid ancestor range to prevent rewrite attacks floor, ceil := int64(-1), d.lightchain.CurrentHeader().Number.Uint64() @@ -622,7 +619,7 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { if count > limit { count = limit } - go p.getAbsHeaders(uint64(from), count, 15, false) + go p.peer.RequestHeadersByNumber(uint64(from), count, 15, false) // Wait for the remote response to the head fetch number, hash := uint64(0), common.Hash{} @@ -704,7 +701,7 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { ttl := d.requestTTL() timeout := time.After(ttl) - go p.getAbsHeaders(uint64(check), 1, 0, false) + go p.peer.RequestHeadersByNumber(uint64(check), 1, 0, false) // Wait until a reply arrives to this request for arrived := false; !arrived; { @@ -765,7 +762,7 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { // other peers are only accepted if they map cleanly to the skeleton. If no one // can fill in the skeleton - not even the origin peer - it's assumed invalid and // the origin is dropped. -func (d *Downloader) fetchHeaders(p *peer, from uint64) error { +func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error { p.log.Debug("Directing header downloads", "origin", from) defer p.log.Debug("Header download terminated") @@ -785,10 +782,10 @@ func (d *Downloader) fetchHeaders(p *peer, from uint64) error { if skeleton { p.log.Trace("Fetching skeleton headers", "count", MaxHeaderFetch, "from", from) - go p.getAbsHeaders(from+uint64(MaxHeaderFetch)-1, MaxSkeletonSize, MaxHeaderFetch-1, false) + go p.peer.RequestHeadersByNumber(from+uint64(MaxHeaderFetch)-1, MaxSkeletonSize, MaxHeaderFetch-1, false) } else { p.log.Trace("Fetching full headers", "count", MaxHeaderFetch, "from", from) - go p.getAbsHeaders(from, MaxHeaderFetch, 0, false) + go p.peer.RequestHeadersByNumber(from, MaxHeaderFetch, 0, false) } } // Start pulling the header chain skeleton until all is done @@ -890,12 +887,12 @@ func (d *Downloader) fillHeaderSkeleton(from uint64, skeleton []*types.Header) ( } expire = func() map[string]int { return d.queue.ExpireHeaders(d.requestTTL()) } throttle = func() bool { return false } - reserve = func(p *peer, count int) (*fetchRequest, bool, error) { + reserve = func(p *peerConnection, count int) (*fetchRequest, bool, error) { return d.queue.ReserveHeaders(p, count), false, nil } - fetch = func(p *peer, req *fetchRequest) error { return p.FetchHeaders(req.From, MaxHeaderFetch) } - capacity = func(p *peer) int { return p.HeaderCapacity(d.requestRTT()) } - setIdle = func(p *peer, accepted int) { p.SetHeadersIdle(accepted) } + fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchHeaders(req.From, MaxHeaderFetch) } + capacity = func(p *peerConnection) int { return p.HeaderCapacity(d.requestRTT()) } + setIdle = func(p *peerConnection, accepted int) { p.SetHeadersIdle(accepted) } ) err := d.fetchParts(errCancelHeaderFetch, d.headerCh, deliver, d.queue.headerContCh, expire, d.queue.PendingHeaders, d.queue.InFlightHeaders, throttle, reserve, @@ -919,9 +916,9 @@ func (d *Downloader) fetchBodies(from uint64) error { return d.queue.DeliverBodies(pack.peerId, pack.transactions, pack.uncles) } expire = func() map[string]int { return d.queue.ExpireBodies(d.requestTTL()) } - fetch = func(p *peer, req *fetchRequest) error { return p.FetchBodies(req) } - capacity = func(p *peer) int { return p.BlockCapacity(d.requestRTT()) } - setIdle = func(p *peer, accepted int) { p.SetBodiesIdle(accepted) } + fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchBodies(req) } + capacity = func(p *peerConnection) int { return p.BlockCapacity(d.requestRTT()) } + setIdle = func(p *peerConnection, accepted int) { p.SetBodiesIdle(accepted) } ) err := d.fetchParts(errCancelBodyFetch, d.bodyCh, deliver, d.bodyWakeCh, expire, d.queue.PendingBlocks, d.queue.InFlightBlocks, d.queue.ShouldThrottleBlocks, d.queue.ReserveBodies, @@ -943,9 +940,9 @@ func (d *Downloader) fetchReceipts(from uint64) error { return d.queue.DeliverReceipts(pack.peerId, pack.receipts) } expire = func() map[string]int { return d.queue.ExpireReceipts(d.requestTTL()) } - fetch = func(p *peer, req *fetchRequest) error { return p.FetchReceipts(req) } - capacity = func(p *peer) int { return p.ReceiptCapacity(d.requestRTT()) } - setIdle = func(p *peer, accepted int) { p.SetReceiptsIdle(accepted) } + fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchReceipts(req) } + capacity = func(p *peerConnection) int { return p.ReceiptCapacity(d.requestRTT()) } + setIdle = func(p *peerConnection, accepted int) { p.SetReceiptsIdle(accepted) } ) err := d.fetchParts(errCancelReceiptFetch, d.receiptCh, deliver, d.receiptWakeCh, expire, d.queue.PendingReceipts, d.queue.InFlightReceipts, d.queue.ShouldThrottleReceipts, d.queue.ReserveReceipts, @@ -981,9 +978,9 @@ func (d *Downloader) fetchReceipts(from uint64) error { // - setIdle: network callback to set a peer back to idle and update its estimated capacity (traffic shaping) // - kind: textual label of the type being downloaded to display in log mesages func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliver func(dataPack) (int, error), wakeCh chan bool, - expire func() map[string]int, pending func() int, inFlight func() bool, throttle func() bool, reserve func(*peer, int) (*fetchRequest, bool, error), - fetchHook func([]*types.Header), fetch func(*peer, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peer) int, - idle func() ([]*peer, int), setIdle func(*peer, int), kind string) error { + expire func() map[string]int, pending func() int, inFlight func() bool, throttle func() bool, reserve func(*peerConnection, int) (*fetchRequest, bool, error), + fetchHook func([]*types.Header), fetch func(*peerConnection, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peerConnection) int, + idle func() ([]*peerConnection, int), setIdle func(*peerConnection, int), kind string) error { // Create a ticker to detect expired retrieval tasks ticker := time.NewTicker(100 * time.Millisecond) @@ -1261,7 +1258,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { if chunk[len(chunk)-1].Number.Uint64()+uint64(fsHeaderForceVerify) > pivot { frequency = 1 } - if n, err := d.chain.InsertHeaderChain(chunk, frequency); err != nil { + if n, err := d.lightchain.InsertHeaderChain(chunk, frequency); err != nil { // If some headers were inserted, add them too to the rollback list if n > 0 { rollback = append(rollback, chunk[:n]...) diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 5852127ea7..bad2851d4a 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -404,14 +404,7 @@ func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Ha defer dl.lock.Unlock() var err error - switch version { - case 62: - err = dl.downloader.RegisterPeer(id, version, dl.peerCurrentHeadFn(id), dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), nil, nil) - case 63: - err = dl.downloader.RegisterPeer(id, version, dl.peerCurrentHeadFn(id), dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay)) - case 64: - err = dl.downloader.RegisterPeer(id, version, dl.peerCurrentHeadFn(id), dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay)) - } + err = dl.downloader.RegisterPeer(id, version, &downloadTesterPeer{dl, id, delay}) if err == nil { // Assign the owned hashes, headers and blocks to the peer (deep copy) dl.peerHashes[id] = make([]common.Hash, len(hashes)) @@ -469,139 +462,133 @@ func (dl *downloadTester) dropPeer(id string) { dl.downloader.UnregisterPeer(id) } -// peerCurrentHeadFn constructs a function to retrieve a peer's current head hash +type downloadTesterPeer struct { + dl *downloadTester + id string + delay time.Duration +} + +// Head constructs a function to retrieve a peer's current head hash // and total difficulty. -func (dl *downloadTester) peerCurrentHeadFn(id string) func() (common.Hash, *big.Int) { - return func() (common.Hash, *big.Int) { - dl.lock.RLock() - defer dl.lock.RUnlock() +func (dlp *downloadTesterPeer) Head() (common.Hash, *big.Int) { + dlp.dl.lock.RLock() + defer dlp.dl.lock.RUnlock() - return dl.peerHashes[id][0], nil - } + return dlp.dl.peerHashes[dlp.id][0], nil } -// peerGetRelHeadersFn constructs a GetBlockHeaders function based on a hashed +// RequestHeadersByHash constructs a GetBlockHeaders function based on a hashed // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. -func (dl *downloadTester) peerGetRelHeadersFn(id string, delay time.Duration) func(common.Hash, int, int, bool) error { - return func(origin common.Hash, amount int, skip int, reverse bool) error { - // Find the canonical number of the hash - dl.lock.RLock() - number := uint64(0) - for num, hash := range dl.peerHashes[id] { - if hash == origin { - number = uint64(len(dl.peerHashes[id]) - num - 1) - break - } +func (dlp *downloadTesterPeer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { + // Find the canonical number of the hash + dlp.dl.lock.RLock() + number := uint64(0) + for num, hash := range dlp.dl.peerHashes[dlp.id] { + if hash == origin { + number = uint64(len(dlp.dl.peerHashes[dlp.id]) - num - 1) + break } - dl.lock.RUnlock() - - // Use the absolute header fetcher to satisfy the query - return dl.peerGetAbsHeadersFn(id, delay)(number, amount, skip, reverse) } + dlp.dl.lock.RUnlock() + + // Use the absolute header fetcher to satisfy the query + return dlp.RequestHeadersByNumber(number, amount, skip, reverse) } -// peerGetAbsHeadersFn constructs a GetBlockHeaders function based on a numbered +// RequestHeadersByNumber constructs a GetBlockHeaders function based on a numbered // origin; associated with a particular peer in the download tester. The returned // function can be used to retrieve batches of headers from the particular peer. -func (dl *downloadTester) peerGetAbsHeadersFn(id string, delay time.Duration) func(uint64, int, int, bool) error { - return func(origin uint64, amount int, skip int, reverse bool) error { - time.Sleep(delay) +func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { + time.Sleep(dlp.delay) - dl.lock.RLock() - defer dl.lock.RUnlock() + dlp.dl.lock.RLock() + defer dlp.dl.lock.RUnlock() - // Gather the next batch of headers - hashes := dl.peerHashes[id] - headers := dl.peerHeaders[id] - result := make([]*types.Header, 0, amount) - for i := 0; i < amount && len(hashes)-int(origin)-1-i*(skip+1) >= 0; i++ { - if header, ok := headers[hashes[len(hashes)-int(origin)-1-i*(skip+1)]]; ok { - result = append(result, header) - } + // Gather the next batch of headers + hashes := dlp.dl.peerHashes[dlp.id] + headers := dlp.dl.peerHeaders[dlp.id] + result := make([]*types.Header, 0, amount) + for i := 0; i < amount && len(hashes)-int(origin)-1-i*(skip+1) >= 0; i++ { + if header, ok := headers[hashes[len(hashes)-int(origin)-1-i*(skip+1)]]; ok { + result = append(result, header) } - // Delay delivery a bit to allow attacks to unfold - go func() { - time.Sleep(time.Millisecond) - dl.downloader.DeliverHeaders(id, result) - }() - return nil } + // Delay delivery a bit to allow attacks to unfold + go func() { + time.Sleep(time.Millisecond) + dlp.dl.downloader.DeliverHeaders(dlp.id, result) + }() + return nil } -// peerGetBodiesFn constructs a getBlockBodies method associated with a particular +// RequestBodies constructs a getBlockBodies method associated with a particular // peer in the download tester. The returned function can be used to retrieve // batches of block bodies from the particularly requested peer. -func (dl *downloadTester) peerGetBodiesFn(id string, delay time.Duration) func([]common.Hash) error { - return func(hashes []common.Hash) error { - time.Sleep(delay) +func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { + time.Sleep(dlp.delay) - dl.lock.RLock() - defer dl.lock.RUnlock() + dlp.dl.lock.RLock() + defer dlp.dl.lock.RUnlock() - blocks := dl.peerBlocks[id] + blocks := dlp.dl.peerBlocks[dlp.id] - transactions := make([][]*types.Transaction, 0, len(hashes)) - uncles := make([][]*types.Header, 0, len(hashes)) + transactions := make([][]*types.Transaction, 0, len(hashes)) + uncles := make([][]*types.Header, 0, len(hashes)) - for _, hash := range hashes { - if block, ok := blocks[hash]; ok { - transactions = append(transactions, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } + for _, hash := range hashes { + if block, ok := blocks[hash]; ok { + transactions = append(transactions, block.Transactions()) + uncles = append(uncles, block.Uncles()) } - go dl.downloader.DeliverBodies(id, transactions, uncles) - - return nil } + go dlp.dl.downloader.DeliverBodies(dlp.id, transactions, uncles) + + return nil } -// peerGetReceiptsFn constructs a getReceipts method associated with a particular +// RequestReceipts constructs a getReceipts method associated with a particular // peer in the download tester. The returned function can be used to retrieve // batches of block receipts from the particularly requested peer. -func (dl *downloadTester) peerGetReceiptsFn(id string, delay time.Duration) func([]common.Hash) error { - return func(hashes []common.Hash) error { - time.Sleep(delay) +func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { + time.Sleep(dlp.delay) - dl.lock.RLock() - defer dl.lock.RUnlock() + dlp.dl.lock.RLock() + defer dlp.dl.lock.RUnlock() - receipts := dl.peerReceipts[id] + receipts := dlp.dl.peerReceipts[dlp.id] - results := make([][]*types.Receipt, 0, len(hashes)) - for _, hash := range hashes { - if receipt, ok := receipts[hash]; ok { - results = append(results, receipt) - } + results := make([][]*types.Receipt, 0, len(hashes)) + for _, hash := range hashes { + if receipt, ok := receipts[hash]; ok { + results = append(results, receipt) } - go dl.downloader.DeliverReceipts(id, results) - - return nil } + go dlp.dl.downloader.DeliverReceipts(dlp.id, results) + + return nil } -// peerGetNodeDataFn constructs a getNodeData method associated with a particular +// RequestNodeData constructs a getNodeData method associated with a particular // peer in the download tester. The returned function can be used to retrieve // batches of node state data from the particularly requested peer. -func (dl *downloadTester) peerGetNodeDataFn(id string, delay time.Duration) func([]common.Hash) error { - return func(hashes []common.Hash) error { - time.Sleep(delay) +func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error { + time.Sleep(dlp.delay) - dl.lock.RLock() - defer dl.lock.RUnlock() + dlp.dl.lock.RLock() + defer dlp.dl.lock.RUnlock() - results := make([][]byte, 0, len(hashes)) - for _, hash := range hashes { - if data, err := dl.peerDb.Get(hash.Bytes()); err == nil { - if !dl.peerMissingStates[id][hash] { - results = append(results, data) - } + results := make([][]byte, 0, len(hashes)) + for _, hash := range hashes { + if data, err := dlp.dl.peerDb.Get(hash.Bytes()); err == nil { + if !dlp.dl.peerMissingStates[dlp.id][hash] { + results = append(results, data) } } - go dl.downloader.DeliverNodeData(id, results) - - return nil } + go dlp.dl.downloader.DeliverNodeData(dlp.id, results) + + return nil } // assertOwnChain checks if the local chain contains the correct number of items @@ -1668,6 +1655,48 @@ func TestDeliverHeadersHang64Full(t *testing.T) { testDeliverHeadersHang(t, 64, func TestDeliverHeadersHang64Fast(t *testing.T) { testDeliverHeadersHang(t, 64, FastSync) } func TestDeliverHeadersHang64Light(t *testing.T) { testDeliverHeadersHang(t, 64, LightSync) } +type floodingTestPeer struct { + peer Peer + tester *downloadTester +} + +func (ftp *floodingTestPeer) Head() (common.Hash, *big.Int) { return ftp.peer.Head() } +func (ftp *floodingTestPeer) RequestHeadersByHash(hash common.Hash, count int, skip int, reverse bool) error { + return ftp.peer.RequestHeadersByHash(hash, count, skip, reverse) +} +func (ftp *floodingTestPeer) RequestBodies(hashes []common.Hash) error { + return ftp.peer.RequestBodies(hashes) +} +func (ftp *floodingTestPeer) RequestReceipts(hashes []common.Hash) error { + return ftp.peer.RequestReceipts(hashes) +} +func (ftp *floodingTestPeer) RequestNodeData(hashes []common.Hash) error { + return ftp.peer.RequestNodeData(hashes) +} + +func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int, reverse bool) error { + deliveriesDone := make(chan struct{}, 500) + for i := 0; i < cap(deliveriesDone); i++ { + peer := fmt.Sprintf("fake-peer%d", i) + go func() { + ftp.tester.downloader.DeliverHeaders(peer, []*types.Header{{}, {}, {}, {}}) + deliveriesDone <- struct{}{} + }() + } + // Deliver the actual requested headers. + go ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) + // None of the extra deliveries should block. + timeout := time.After(15 * time.Second) + for i := 0; i < cap(deliveriesDone); i++ { + select { + case <-deliveriesDone: + case <-timeout: + panic("blocked") + } + } + return nil +} + func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { t.Parallel() @@ -1675,7 +1704,6 @@ func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { defer master.terminate() hashes, headers, blocks, receipts := master.makeChain(5, 0, master.genesis, nil, false) - fakeHeads := []*types.Header{{}, {}, {}, {}} for i := 0; i < 200; i++ { tester := newTester() tester.peerDb = master.peerDb @@ -1683,29 +1711,11 @@ func testDeliverHeadersHang(t *testing.T, protocol int, mode SyncMode) { tester.newPeer("peer", protocol, hashes, headers, blocks, receipts) // Whenever the downloader requests headers, flood it with // a lot of unrequested header deliveries. - tester.downloader.peers.peers["peer"].getAbsHeaders = func(from uint64, count, skip int, reverse bool) error { - deliveriesDone := make(chan struct{}, 500) - for i := 0; i < cap(deliveriesDone); i++ { - peer := fmt.Sprintf("fake-peer%d", i) - go func() { - tester.downloader.DeliverHeaders(peer, fakeHeads) - deliveriesDone <- struct{}{} - }() - } - // Deliver the actual requested headers. - impl := tester.peerGetAbsHeadersFn("peer", 0) - go impl(from, count, skip, reverse) - // None of the extra deliveries should block. - timeout := time.After(15 * time.Second) - for i := 0; i < cap(deliveriesDone); i++ { - select { - case <-deliveriesDone: - case <-timeout: - panic("blocked") - } - } - return nil + tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ + tester.downloader.peers.peers["peer"].peer, + tester, } + if err := tester.sync("peer", nil, mode); err != nil { t.Errorf("sync failed: %v", err) } @@ -1737,7 +1747,7 @@ func testFastCriticalRestarts(t *testing.T, protocol int, progress bool) { for i := 0; i < fsPivotInterval; i++ { tester.peerMissingStates["peer"][headers[hashes[fsMinFullBlocks+i]].Root] = true } - tester.downloader.peers.peers["peer"].getNodeData = tester.peerGetNodeDataFn("peer", 500*time.Millisecond) // Enough to reach the critical section + (tester.downloader.peers.peers["peer"].peer).(*downloadTesterPeer).delay = 500 * time.Millisecond // Enough to reach the critical section // Synchronise with the peer a few times and make sure they fail until the retry limit for i := 0; i < int(fsCriticalTrials)-1; i++ { @@ -1756,7 +1766,7 @@ func testFastCriticalRestarts(t *testing.T, protocol int, progress bool) { tester.lock.Lock() tester.peerHeaders["peer"][hashes[fsMinFullBlocks-1]] = headers[hashes[fsMinFullBlocks-1]] tester.peerMissingStates["peer"] = map[common.Hash]bool{tester.downloader.fsPivotLock.Root: true} - tester.downloader.peers.peers["peer"].getNodeData = tester.peerGetNodeDataFn("peer", 0) + (tester.downloader.peers.peers["peer"].peer).(*downloadTesterPeer).delay = 0 tester.lock.Unlock() } } diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index dc8b097728..71a6926883 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -39,16 +39,6 @@ const ( measurementImpact = 0.1 // The impact a single measurement has on a peer's final throughput value. ) -// Head hash and total difficulty retriever for -type currentHeadRetrievalFn func() (common.Hash, *big.Int) - -// Block header and body fetchers belonging to eth/62 and above -type relativeHeaderFetcherFn func(common.Hash, int, int, bool) error -type absoluteHeaderFetcherFn func(uint64, int, int, bool) error -type blockBodyFetcherFn func([]common.Hash) error -type receiptFetcherFn func([]common.Hash) error -type stateFetcherFn func([]common.Hash) error - var ( errAlreadyFetching = errors.New("already fetching blocks from peer") errAlreadyRegistered = errors.New("peer is already registered") @@ -56,7 +46,7 @@ var ( ) // peer represents an active peer from which hashes and blocks are retrieved. -type peer struct { +type peerConnection struct { id string // Unique identifier of the peer headerIdle int32 // Current header activity state of the peer (idle = 0, active = 1) @@ -78,37 +68,31 @@ type peer struct { lacking map[common.Hash]struct{} // Set of hashes not to request (didn't have previously) - currentHead currentHeadRetrievalFn // Method to fetch the currently known head of the peer - - getRelHeaders relativeHeaderFetcherFn // [eth/62] Method to retrieve a batch of headers from an origin hash - getAbsHeaders absoluteHeaderFetcherFn // [eth/62] Method to retrieve a batch of headers from an absolute position - getBlockBodies blockBodyFetcherFn // [eth/62] Method to retrieve a batch of block bodies - - getReceipts receiptFetcherFn // [eth/63] Method to retrieve a batch of block transaction receipts - getNodeData stateFetcherFn // [eth/63] Method to retrieve a batch of state trie data + peer Peer version int // Eth protocol version number to switch strategies log log.Logger // Contextual logger to add extra infos to peer logs lock sync.RWMutex } +type Peer interface { + Head() (common.Hash, *big.Int) + RequestHeadersByHash(common.Hash, int, int, bool) error + RequestHeadersByNumber(uint64, int, int, bool) error + RequestBodies([]common.Hash) error + RequestReceipts([]common.Hash) error + RequestNodeData([]common.Hash) error +} + // newPeer create a new downloader peer, with specific hash and block retrieval // mechanisms. -func newPeer(id string, version int, currentHead currentHeadRetrievalFn, - getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn, - getReceipts receiptFetcherFn, getNodeData stateFetcherFn, logger log.Logger) *peer { +func newPeerConnection(id string, version int, peer Peer, logger log.Logger) *peerConnection { - return &peer{ + return &peerConnection{ id: id, lacking: make(map[common.Hash]struct{}), - currentHead: currentHead, - getRelHeaders: getRelHeaders, - getAbsHeaders: getAbsHeaders, - getBlockBodies: getBlockBodies, - - getReceipts: getReceipts, - getNodeData: getNodeData, + peer: peer, version: version, log: logger, @@ -116,7 +100,7 @@ func newPeer(id string, version int, currentHead currentHeadRetrievalFn, } // Reset clears the internal state of a peer entity. -func (p *peer) Reset() { +func (p *peerConnection) Reset() { p.lock.Lock() defer p.lock.Unlock() @@ -134,7 +118,7 @@ func (p *peer) Reset() { } // FetchHeaders sends a header retrieval request to the remote peer. -func (p *peer) FetchHeaders(from uint64, count int) error { +func (p *peerConnection) FetchHeaders(from uint64, count int) error { // Sanity check the protocol version if p.version < 62 { panic(fmt.Sprintf("header fetch [eth/62+] requested on eth/%d", p.version)) @@ -146,13 +130,13 @@ func (p *peer) FetchHeaders(from uint64, count int) error { p.headerStarted = time.Now() // Issue the header retrieval request (absolut upwards without gaps) - go p.getAbsHeaders(from, count, 0, false) + go p.peer.RequestHeadersByNumber(from, count, 0, false) return nil } // FetchBodies sends a block body retrieval request to the remote peer. -func (p *peer) FetchBodies(request *fetchRequest) error { +func (p *peerConnection) FetchBodies(request *fetchRequest) error { // Sanity check the protocol version if p.version < 62 { panic(fmt.Sprintf("body fetch [eth/62+] requested on eth/%d", p.version)) @@ -168,13 +152,13 @@ func (p *peer) FetchBodies(request *fetchRequest) error { for _, header := range request.Headers { hashes = append(hashes, header.Hash()) } - go p.getBlockBodies(hashes) + go p.peer.RequestBodies(hashes) return nil } // FetchReceipts sends a receipt retrieval request to the remote peer. -func (p *peer) FetchReceipts(request *fetchRequest) error { +func (p *peerConnection) FetchReceipts(request *fetchRequest) error { // Sanity check the protocol version if p.version < 63 { panic(fmt.Sprintf("body fetch [eth/63+] requested on eth/%d", p.version)) @@ -190,13 +174,13 @@ func (p *peer) FetchReceipts(request *fetchRequest) error { for _, header := range request.Headers { hashes = append(hashes, header.Hash()) } - go p.getReceipts(hashes) + go p.peer.RequestReceipts(hashes) return nil } // FetchNodeData sends a node state data retrieval request to the remote peer. -func (p *peer) FetchNodeData(hashes []common.Hash) error { +func (p *peerConnection) FetchNodeData(hashes []common.Hash) error { // Sanity check the protocol version if p.version < 63 { panic(fmt.Sprintf("node data fetch [eth/63+] requested on eth/%d", p.version)) @@ -206,48 +190,50 @@ func (p *peer) FetchNodeData(hashes []common.Hash) error { return errAlreadyFetching } p.stateStarted = time.Now() - go p.getNodeData(hashes) + + go p.peer.RequestNodeData(hashes) + return nil } // SetHeadersIdle sets the peer to idle, allowing it to execute new header retrieval // requests. Its estimated header retrieval throughput is updated with that measured // just now. -func (p *peer) SetHeadersIdle(delivered int) { +func (p *peerConnection) SetHeadersIdle(delivered int) { p.setIdle(p.headerStarted, delivered, &p.headerThroughput, &p.headerIdle) } // SetBlocksIdle sets the peer to idle, allowing it to execute new block retrieval // requests. Its estimated block retrieval throughput is updated with that measured // just now. -func (p *peer) SetBlocksIdle(delivered int) { +func (p *peerConnection) SetBlocksIdle(delivered int) { p.setIdle(p.blockStarted, delivered, &p.blockThroughput, &p.blockIdle) } // SetBodiesIdle sets the peer to idle, allowing it to execute block body retrieval // requests. Its estimated body retrieval throughput is updated with that measured // just now. -func (p *peer) SetBodiesIdle(delivered int) { +func (p *peerConnection) SetBodiesIdle(delivered int) { p.setIdle(p.blockStarted, delivered, &p.blockThroughput, &p.blockIdle) } // SetReceiptsIdle sets the peer to idle, allowing it to execute new receipt // retrieval requests. Its estimated receipt retrieval throughput is updated // with that measured just now. -func (p *peer) SetReceiptsIdle(delivered int) { +func (p *peerConnection) SetReceiptsIdle(delivered int) { p.setIdle(p.receiptStarted, delivered, &p.receiptThroughput, &p.receiptIdle) } // SetNodeDataIdle sets the peer to idle, allowing it to execute new state trie // data retrieval requests. Its estimated state retrieval throughput is updated // with that measured just now. -func (p *peer) SetNodeDataIdle(delivered int) { +func (p *peerConnection) SetNodeDataIdle(delivered int) { p.setIdle(p.stateStarted, delivered, &p.stateThroughput, &p.stateIdle) } // setIdle sets the peer to idle, allowing it to execute new retrieval requests. // Its estimated retrieval throughput is updated with that measured just now. -func (p *peer) setIdle(started time.Time, delivered int, throughput *float64, idle *int32) { +func (p *peerConnection) setIdle(started time.Time, delivered int, throughput *float64, idle *int32) { // Irrelevant of the scaling, make sure the peer ends up idle defer atomic.StoreInt32(idle, 0) @@ -274,7 +260,7 @@ func (p *peer) setIdle(started time.Time, delivered int, throughput *float64, id // HeaderCapacity retrieves the peers header download allowance based on its // previously discovered throughput. -func (p *peer) HeaderCapacity(targetRTT time.Duration) int { +func (p *peerConnection) HeaderCapacity(targetRTT time.Duration) int { p.lock.RLock() defer p.lock.RUnlock() @@ -283,7 +269,7 @@ func (p *peer) HeaderCapacity(targetRTT time.Duration) int { // BlockCapacity retrieves the peers block download allowance based on its // previously discovered throughput. -func (p *peer) BlockCapacity(targetRTT time.Duration) int { +func (p *peerConnection) BlockCapacity(targetRTT time.Duration) int { p.lock.RLock() defer p.lock.RUnlock() @@ -292,7 +278,7 @@ func (p *peer) BlockCapacity(targetRTT time.Duration) int { // ReceiptCapacity retrieves the peers receipt download allowance based on its // previously discovered throughput. -func (p *peer) ReceiptCapacity(targetRTT time.Duration) int { +func (p *peerConnection) ReceiptCapacity(targetRTT time.Duration) int { p.lock.RLock() defer p.lock.RUnlock() @@ -301,7 +287,7 @@ func (p *peer) ReceiptCapacity(targetRTT time.Duration) int { // NodeDataCapacity retrieves the peers state download allowance based on its // previously discovered throughput. -func (p *peer) NodeDataCapacity(targetRTT time.Duration) int { +func (p *peerConnection) NodeDataCapacity(targetRTT time.Duration) int { p.lock.RLock() defer p.lock.RUnlock() @@ -311,7 +297,7 @@ func (p *peer) NodeDataCapacity(targetRTT time.Duration) int { // MarkLacking appends a new entity to the set of items (blocks, receipts, states) // that a peer is known not to have (i.e. have been requested before). If the // set reaches its maximum allowed capacity, items are randomly dropped off. -func (p *peer) MarkLacking(hash common.Hash) { +func (p *peerConnection) MarkLacking(hash common.Hash) { p.lock.Lock() defer p.lock.Unlock() @@ -326,7 +312,7 @@ func (p *peer) MarkLacking(hash common.Hash) { // Lacks retrieves whether the hash of a blockchain item is on the peers lacking // list (i.e. whether we know that the peer does not have it). -func (p *peer) Lacks(hash common.Hash) bool { +func (p *peerConnection) Lacks(hash common.Hash) bool { p.lock.RLock() defer p.lock.RUnlock() @@ -337,7 +323,7 @@ func (p *peer) Lacks(hash common.Hash) bool { // peerSet represents the collection of active peer participating in the chain // download procedure. type peerSet struct { - peers map[string]*peer + peers map[string]*peerConnection newPeerFeed event.Feed lock sync.RWMutex } @@ -345,11 +331,11 @@ type peerSet struct { // newPeerSet creates a new peer set top track the active download sources. func newPeerSet() *peerSet { return &peerSet{ - peers: make(map[string]*peer), + peers: make(map[string]*peerConnection), } } -func (ps *peerSet) SubscribeNewPeers(ch chan<- *peer) event.Subscription { +func (ps *peerSet) SubscribeNewPeers(ch chan<- *peerConnection) event.Subscription { return ps.newPeerFeed.Subscribe(ch) } @@ -370,7 +356,7 @@ func (ps *peerSet) Reset() { // The method also sets the starting throughput values of the new peer to the // average of all existing peers, to give it a realistic chance of being used // for data retrievals. -func (ps *peerSet) Register(p *peer) error { +func (ps *peerSet) Register(p *peerConnection) error { // Retrieve the current median RTT as a sane default p.rtt = ps.medianRTT() @@ -417,7 +403,7 @@ func (ps *peerSet) Unregister(id string) error { } // Peer retrieves the registered peer with the given id. -func (ps *peerSet) Peer(id string) *peer { +func (ps *peerSet) Peer(id string) *peerConnection { ps.lock.RLock() defer ps.lock.RUnlock() @@ -433,11 +419,11 @@ func (ps *peerSet) Len() int { } // AllPeers retrieves a flat list of all the peers within the set. -func (ps *peerSet) AllPeers() []*peer { +func (ps *peerSet) AllPeers() []*peerConnection { ps.lock.RLock() defer ps.lock.RUnlock() - list := make([]*peer, 0, len(ps.peers)) + list := make([]*peerConnection, 0, len(ps.peers)) for _, p := range ps.peers { list = append(list, p) } @@ -446,11 +432,11 @@ func (ps *peerSet) AllPeers() []*peer { // HeaderIdlePeers retrieves a flat list of all the currently header-idle peers // within the active peer set, ordered by their reputation. -func (ps *peerSet) HeaderIdlePeers() ([]*peer, int) { - idle := func(p *peer) bool { +func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { + idle := func(p *peerConnection) bool { return atomic.LoadInt32(&p.headerIdle) == 0 } - throughput := func(p *peer) float64 { + throughput := func(p *peerConnection) float64 { p.lock.RLock() defer p.lock.RUnlock() return p.headerThroughput @@ -460,11 +446,11 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peer, int) { // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within // the active peer set, ordered by their reputation. -func (ps *peerSet) BodyIdlePeers() ([]*peer, int) { - idle := func(p *peer) bool { +func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { + idle := func(p *peerConnection) bool { return atomic.LoadInt32(&p.blockIdle) == 0 } - throughput := func(p *peer) float64 { + throughput := func(p *peerConnection) float64 { p.lock.RLock() defer p.lock.RUnlock() return p.blockThroughput @@ -474,11 +460,11 @@ func (ps *peerSet) BodyIdlePeers() ([]*peer, int) { // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers // within the active peer set, ordered by their reputation. -func (ps *peerSet) ReceiptIdlePeers() ([]*peer, int) { - idle := func(p *peer) bool { +func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { + idle := func(p *peerConnection) bool { return atomic.LoadInt32(&p.receiptIdle) == 0 } - throughput := func(p *peer) float64 { + throughput := func(p *peerConnection) float64 { p.lock.RLock() defer p.lock.RUnlock() return p.receiptThroughput @@ -488,11 +474,11 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peer, int) { // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle // peers within the active peer set, ordered by their reputation. -func (ps *peerSet) NodeDataIdlePeers() ([]*peer, int) { - idle := func(p *peer) bool { +func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { + idle := func(p *peerConnection) bool { return atomic.LoadInt32(&p.stateIdle) == 0 } - throughput := func(p *peer) float64 { + throughput := func(p *peerConnection) float64 { p.lock.RLock() defer p.lock.RUnlock() return p.stateThroughput @@ -503,11 +489,11 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peer, int) { // idlePeers retrieves a flat list of all currently idle peers satisfying the // protocol version constraints, using the provided function to check idleness. // The resulting set of peers are sorted by their measure throughput. -func (ps *peerSet) idlePeers(minProtocol, maxProtocol int, idleCheck func(*peer) bool, throughput func(*peer) float64) ([]*peer, int) { +func (ps *peerSet) idlePeers(minProtocol, maxProtocol int, idleCheck func(*peerConnection) bool, throughput func(*peerConnection) float64) ([]*peerConnection, int) { ps.lock.RLock() defer ps.lock.RUnlock() - idle, total := make([]*peer, 0, len(ps.peers)), 0 + idle, total := make([]*peerConnection, 0, len(ps.peers)), 0 for _, p := range ps.peers { if p.version >= minProtocol && p.version <= maxProtocol { if idleCheck(p) { diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 8a7735d673..6926f1d8c8 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -41,7 +41,7 @@ var ( // fetchRequest is a currently running data retrieval operation. type fetchRequest struct { - Peer *peer // Peer to which the request was sent + Peer *peerConnection // Peer to which the request was sent From uint64 // [eth/62] Requested chain element index (used for skeleton fills only) Hashes map[common.Hash]int // [eth/61] Requested hashes with their insertion index (priority) Headers []*types.Header // [eth/62] Requested headers, sorted by request order @@ -391,7 +391,7 @@ func (q *queue) countProcessableItems() int { // ReserveHeaders reserves a set of headers for the given peer, skipping any // previously failed batches. -func (q *queue) ReserveHeaders(p *peer, count int) *fetchRequest { +func (q *queue) ReserveHeaders(p *peerConnection, count int) *fetchRequest { q.lock.Lock() defer q.lock.Unlock() @@ -432,7 +432,7 @@ func (q *queue) ReserveHeaders(p *peer, count int) *fetchRequest { // ReserveBodies reserves a set of body fetches for the given peer, skipping any // previously failed downloads. Beside the next batch of needed fetches, it also // returns a flag whether empty blocks were queued requiring processing. -func (q *queue) ReserveBodies(p *peer, count int) (*fetchRequest, bool, error) { +func (q *queue) ReserveBodies(p *peerConnection, count int) (*fetchRequest, bool, error) { isNoop := func(header *types.Header) bool { return header.TxHash == types.EmptyRootHash && header.UncleHash == types.EmptyUncleHash } @@ -445,7 +445,7 @@ func (q *queue) ReserveBodies(p *peer, count int) (*fetchRequest, bool, error) { // ReserveReceipts reserves a set of receipt fetches for the given peer, skipping // any previously failed downloads. Beside the next batch of needed fetches, it // also returns a flag whether empty receipts were queued requiring importing. -func (q *queue) ReserveReceipts(p *peer, count int) (*fetchRequest, bool, error) { +func (q *queue) ReserveReceipts(p *peerConnection, count int) (*fetchRequest, bool, error) { isNoop := func(header *types.Header) bool { return header.ReceiptHash == types.EmptyRootHash } @@ -462,7 +462,7 @@ func (q *queue) ReserveReceipts(p *peer, count int) (*fetchRequest, bool, error) // Note, this method expects the queue lock to be already held for writing. The // reason the lock is not obtained in here is because the parameters already need // to access the queue, so they already need a lock anyway. -func (q *queue) reserveHeaders(p *peer, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, +func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, pendPool map[string]*fetchRequest, donePool map[common.Hash]struct{}, isNoop func(*types.Header) bool) (*fetchRequest, bool, error) { // Short circuit if the pool has been depleted, or if the peer's already // downloading something (sanity check not to corrupt state) diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index 4e66120393..a5ce8c42df 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -37,7 +37,7 @@ type stateReq struct { tasks map[common.Hash]*stateTask // Download tasks to track previous attempts timeout time.Duration // Maximum round trip time for this to complete timer *time.Timer // Timer to fire when the RTT timeout expires - peer *peer // Peer that we're requesting from + peer *peerConnection // Peer that we're requesting from response [][]byte // Response data of the peer (nil for timeouts) } @@ -246,7 +246,7 @@ func (s *stateSync) Cancel() error { // and timeouts. func (s *stateSync) loop() error { // Listen for new peer events to assign tasks to them - newPeer := make(chan *peer, 1024) + newPeer := make(chan *peerConnection, 1024) peerSub := s.d.peers.SubscribeNewPeers(newPeer) defer peerSub.Unsubscribe() diff --git a/eth/handler.go b/eth/handler.go index 8d026c7757..b2422d71c4 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -265,7 +265,7 @@ func (pm *ProtocolManager) handle(p *peer) error { defer pm.removePeer(p.id) // Register the peer in the downloader. If the downloader considers it banned, we disconnect - if err := pm.downloader.RegisterPeer(p.id, p.version, p.Head, p.RequestHeadersByHash, p.RequestHeadersByNumber, p.RequestBodies, p.RequestReceipts, p.RequestNodeData); err != nil { + if err := pm.downloader.RegisterPeer(p.id, p.version, p); err != nil { return err } // Propagate existing transactions. new transactions appearing diff --git a/les/handler.go b/les/handler.go index afd4356efe..967b315219 100644 --- a/les/handler.go +++ b/les/handler.go @@ -838,57 +838,83 @@ func (self *ProtocolManager) NodeInfo() *eth.EthNodeInfo { // downloaderPeerNotify implements peerSetNotify type downloaderPeerNotify ProtocolManager +type peerConnection struct { + manager *ProtocolManager + peer *peer +} + +func (pc *peerConnection) Head() (common.Hash, *big.Int) { + return pc.peer.HeadAndTd() +} + +func (pc *peerConnection) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { + reqID := genReqID() + rq := &distReq{ + getCost: func(dp distPeer) uint64 { + peer := dp.(*peer) + return peer.GetRequestCost(GetBlockHeadersMsg, amount) + }, + canSend: func(dp distPeer) bool { + return dp.(*peer) == pc.peer + }, + request: func(dp distPeer) func() { + peer := dp.(*peer) + cost := peer.GetRequestCost(GetBlockHeadersMsg, amount) + peer.fcServer.QueueRequest(reqID, cost) + return func() { peer.RequestHeadersByHash(reqID, cost, origin, amount, skip, reverse) } + }, + } + _, ok := <-pc.manager.reqDist.queue(rq) + if !ok { + return ErrNoPeers + } + return nil +} + +func (pc *peerConnection) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { + reqID := genReqID() + rq := &distReq{ + getCost: func(dp distPeer) uint64 { + peer := dp.(*peer) + return peer.GetRequestCost(GetBlockHeadersMsg, amount) + }, + canSend: func(dp distPeer) bool { + return dp.(*peer) == pc.peer + }, + request: func(dp distPeer) func() { + peer := dp.(*peer) + cost := peer.GetRequestCost(GetBlockHeadersMsg, amount) + peer.fcServer.QueueRequest(reqID, cost) + return func() { peer.RequestHeadersByNumber(reqID, cost, origin, amount, skip, reverse) } + }, + } + _, ok := <-pc.manager.reqDist.queue(rq) + if !ok { + return ErrNoPeers + } + return nil +} + +func (pc *peerConnection) RequestBodies(hashes []common.Hash) error { + panic("RequestBodies not supported in light client mode sync") + +} +func (pc *peerConnection) RequestReceipts(hashes []common.Hash) error { + panic("RequestReceipts not supported in light client mode sync") + +} + +func (pc *peerConnection) RequestNodeData(hashes []common.Hash) error { + panic("RequestNodeData not supported in light client mode sync") +} + func (d *downloaderPeerNotify) registerPeer(p *peer) { pm := (*ProtocolManager)(d) - - requestHeadersByHash := func(origin common.Hash, amount int, skip int, reverse bool) error { - reqID := genReqID() - rq := &distReq{ - getCost: func(dp distPeer) uint64 { - peer := dp.(*peer) - return peer.GetRequestCost(GetBlockHeadersMsg, amount) - }, - canSend: func(dp distPeer) bool { - return dp.(*peer) == p - }, - request: func(dp distPeer) func() { - peer := dp.(*peer) - cost := peer.GetRequestCost(GetBlockHeadersMsg, amount) - peer.fcServer.QueueRequest(reqID, cost) - return func() { peer.RequestHeadersByHash(reqID, cost, origin, amount, skip, reverse) } - }, - } - _, ok := <-pm.reqDist.queue(rq) - if !ok { - return ErrNoPeers - } - return nil + pc := &peerConnection{ + manager: pm, + peer: p, } - requestHeadersByNumber := func(origin uint64, amount int, skip int, reverse bool) error { - reqID := genReqID() - rq := &distReq{ - getCost: func(dp distPeer) uint64 { - peer := dp.(*peer) - return peer.GetRequestCost(GetBlockHeadersMsg, amount) - }, - canSend: func(dp distPeer) bool { - return dp.(*peer) == p - }, - request: func(dp distPeer) func() { - peer := dp.(*peer) - cost := peer.GetRequestCost(GetBlockHeadersMsg, amount) - peer.fcServer.QueueRequest(reqID, cost) - return func() { peer.RequestHeadersByNumber(reqID, cost, origin, amount, skip, reverse) } - }, - } - _, ok := <-pm.reqDist.queue(rq) - if !ok { - return ErrNoPeers - } - return nil - } - - pm.downloader.RegisterPeer(p.id, ethVersion, p.HeadAndTd, requestHeadersByHash, requestHeadersByNumber, nil, nil, nil) + pm.downloader.RegisterPeer(p.id, ethVersion, pc) } func (d *downloaderPeerNotify) unregisterPeer(p *peer) { From 8bbd598ef43be51f6dbd37b7d2b14501c8a4a6c3 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Jun 2017 14:19:10 +0200 Subject: [PATCH 29/43] core: fix an off-by-one when the block import counts blocks --- core/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index aab2e72f3a..6772ea2848 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1073,7 +1073,7 @@ func (st *insertStats) report(chain []*types.Block, index int) { } log.Info("Imported new chain segment", context...) - *st = insertStats{startTime: now, lastIndex: index} + *st = insertStats{startTime: now, lastIndex: index + 1} } } From c7041fe1455541e59879df2b2d36984d49d1630b Mon Sep 17 00:00:00 2001 From: Lewis Marshall Date: Fri, 30 Jun 2017 10:10:41 +0100 Subject: [PATCH 30/43] cmd/swarm: Exit if --ethapi is set The previous attempt to use --ethapi as a fallback if --ens-api is not set does not work because --ens-api has a default value, and also setting --ens-api to "" is the suggested way to disable ENS lookups. Signed-off-by: Lewis Marshall --- cmd/swarm/main.go | 27 +++++++++++++-------------- cmd/swarm/run_test.go | 2 +- swarm/dev/scripts/boot-cluster.sh | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index f0680c487e..6b4feb0bcc 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -100,10 +100,6 @@ var ( Name: "sync", Usage: "Swarm Syncing enabled (default true)", } - EthAPIFlag = cli.StringFlag{ - Name: "ethapi", - Usage: "DEPRECATED: please use --ens-api and --swap-api", - } EnsAPIFlag = cli.StringFlag{ Name: "ens-api", Usage: "URL of the Ethereum API provider to use for ENS record lookups", @@ -142,6 +138,12 @@ var ( Name: "corsdomain", Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", } + + // the following flags are deprecated and should be removed in the future + DeprecatedEthAPIFlag = cli.StringFlag{ + Name: "ethapi", + Usage: "DEPRECATED: please use --ens-api and --swap-api", + } ) var defaultNodeConfig = node.DefaultConfig @@ -266,7 +268,6 @@ Cleans database of corrupted entries. utils.PasswordFileFlag, // bzzd-specific flags CorsStringFlag, - EthAPIFlag, EnsAPIFlag, EnsAddrFlag, SwarmConfigPathFlag, @@ -285,6 +286,8 @@ Cleans database of corrupted entries. SwarmUploadDefaultPath, SwarmUpFromStdinFlag, SwarmUploadMimeType, + //deprecated flags + DeprecatedEthAPIFlag, } app.Flags = append(app.Flags, debug.Flags...) app.Before = func(ctx *cli.Context) error { @@ -319,6 +322,11 @@ func version(ctx *cli.Context) error { } func bzzd(ctx *cli.Context) error { + // exit if the deprecated --ethapi flag is set + if ctx.GlobalString(DeprecatedEthAPIFlag.Name) != "" { + utils.Fatalf("--ethapi is no longer a valid command line flag, please use --ens-api and/or --swap-api.") + } + cfg := defaultNodeConfig utils.SetNodeConfig(ctx, &cfg) stack, err := node.New(&cfg) @@ -408,21 +416,12 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { swapEnabled := ctx.GlobalBool(SwarmSwapEnabledFlag.Name) syncEnabled := ctx.GlobalBoolT(SwarmSyncEnabledFlag.Name) - ethapi := ctx.GlobalString(EthAPIFlag.Name) - if ethapi != "" { - log.Warn("DEPRECATED: --ethapi is deprecated and will be removed in a future version, please use --ens-api and --swap-api") - } - swapapi := ctx.GlobalString(SwarmSwapAPIFlag.Name) if swapEnabled && swapapi == "" { utils.Fatalf("SWAP is enabled but --swap-api is not set") } ensapi := ctx.GlobalString(EnsAPIFlag.Name) - // use the deprecated --ethapi if --ens-api is not set - if ensapi == "" { - ensapi = ethapi - } ensAddr := ctx.GlobalString(EnsAddrFlag.Name) cors := ctx.GlobalString(CorsStringFlag.Name) diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 2d32a51c8d..05cbb27f1d 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -194,7 +194,7 @@ func newTestNode(t *testing.T, dir string) *testNode { "--nodiscover", "--datadir", dir, "--ipcpath", conf.IPCPath, - "--ethapi", "", + "--ens-api", "", "--bzzaccount", account.Address.String(), "--bzznetworkid", "321", "--bzzport", httpPort, diff --git a/swarm/dev/scripts/boot-cluster.sh b/swarm/dev/scripts/boot-cluster.sh index 073b095ad4..98ae3c8023 100755 --- a/swarm/dev/scripts/boot-cluster.sh +++ b/swarm/dev/scripts/boot-cluster.sh @@ -208,7 +208,7 @@ start_swarm_node() { --bootnodes "${BOOTNODE_URL}" --datadir "${dir}" --identity "${name}" - --ethapi "${GETH_RPC_URL}" + --ens-api "${GETH_RPC_URL}" --bzznetworkid "321" --bzzaccount "${address}" --password "${dir}/password" From 67aff49822a411611941e4b93a0343df75fd21b7 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 30 Jun 2017 22:43:26 +0200 Subject: [PATCH 31/43] core: Change local-handling to use sender-account instead of tx hashes --- core/tx_list.go | 12 +++--- core/tx_pool.go | 97 ++++++++++++++++++++++--------------------------- 2 files changed, 49 insertions(+), 60 deletions(-) diff --git a/core/tx_list.go b/core/tx_list.go index 626d3a3b71..e12af4a891 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -422,7 +422,7 @@ func (l *txPricedList) Removed() { // Discard finds all the transactions below the given price threshold, drops them // from the priced list and returs them for further removal from the entire pool. -func (l *txPricedList) Cap(threshold *big.Int, local *txSet) types.Transactions { +func (l *txPricedList) Cap(threshold *big.Int, local *accountSet) types.Transactions { drop := make(types.Transactions, 0, 128) // Remote underpriced transactions to drop save := make(types.Transactions, 0, 64) // Local underpriced transactions to keep @@ -440,7 +440,7 @@ func (l *txPricedList) Cap(threshold *big.Int, local *txSet) types.Transactions break } // Non stale transaction found, discard unless local - if local.contains(hash) { + if local.contains(tx) { save = append(save, tx) } else { drop = append(drop, tx) @@ -454,9 +454,9 @@ func (l *txPricedList) Cap(threshold *big.Int, local *txSet) types.Transactions // Underpriced checks whether a transaction is cheaper than (or as cheap as) the // lowest priced transaction currently being tracked. -func (l *txPricedList) Underpriced(tx *types.Transaction, local *txSet) bool { +func (l *txPricedList) Underpriced(tx *types.Transaction, local *accountSet) bool { // Local transactions cannot be underpriced - if local.contains(tx.Hash()) { + if local.contains(tx) { return false } // Discard stale price points if found at the heap start @@ -480,7 +480,7 @@ func (l *txPricedList) Underpriced(tx *types.Transaction, local *txSet) bool { // Discard finds a number of most underpriced transactions, removes them from the // priced list and returs them for further removal from the entire pool. -func (l *txPricedList) Discard(count int, local *txSet) types.Transactions { +func (l *txPricedList) Discard(count int, local *accountSet) types.Transactions { drop := make(types.Transactions, 0, count) // Remote underpriced transactions to drop save := make(types.Transactions, 0, 64) // Local underpriced transactions to keep @@ -494,7 +494,7 @@ func (l *txPricedList) Discard(count int, local *txSet) types.Transactions { continue } // Non stale transaction found, discard unless local - if local.contains(hash) { + if local.contains(tx) { save = append(save, tx) } else { drop = append(drop, tx) diff --git a/core/tx_pool.go b/core/tx_pool.go index 2f3cd1e93d..3f758957a7 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -155,7 +155,7 @@ type TxPool struct { gasPrice *big.Int eventMux *event.TypeMux events *event.TypeMuxSubscription - locals *txSet + locals *accountSet signer types.Signer mu sync.RWMutex @@ -176,12 +176,12 @@ type TxPool struct { func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, eventMux *event.TypeMux, currentStateFn stateFn, gasLimitFn func() *big.Int) *TxPool { // Sanitize the input to ensure no vulnerable gas prices are set config = (&config).sanitize() - + signer := types.NewEIP155Signer(chainconfig.ChainId) // Create the transaction pool with its initial settings pool := &TxPool{ config: config, chainconfig: chainconfig, - signer: types.NewEIP155Signer(chainconfig.ChainId), + signer: signer, pending: make(map[common.Address]*txList), queue: make(map[common.Address]*txList), beats: make(map[common.Address]time.Time), @@ -191,7 +191,7 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, eventMux *e gasLimit: gasLimitFn, gasPrice: new(big.Int).SetUint64(config.PriceLimit), pendingState: nil, - locals: newTxSet(), + locals: newAccountSet(signer), events: eventMux.Subscribe(ChainHeadEvent{}, RemovedTransactionEvent{}), quit: make(chan struct{}), } @@ -376,13 +376,19 @@ func (pool *TxPool) Pending() (map[common.Address]types.Transactions, error) { func (pool *TxPool) SetLocal(tx *types.Transaction) { pool.mu.Lock() defer pool.mu.Unlock() - pool.locals.add(tx.Hash()) + pool.locals.add(tx) } // validateTx checks whether a transaction is valid according // to the consensus rules. func (pool *TxPool) validateTx(tx *types.Transaction) error { - local := pool.locals.contains(tx.Hash()) + + from, err := types.Sender(pool.signer, tx) + if err != nil { + return ErrInvalidSender + } + + local := pool.locals.containsAddress(from) // Drop transactions under our own minimal accepted gas price if !local && pool.gasPrice.Cmp(tx.GasPrice()) > 0 { return ErrUnderpriced @@ -393,10 +399,6 @@ func (pool *TxPool) validateTx(tx *types.Transaction) error { return err } - from, err := types.Sender(pool.signer, tx) - if err != nil { - return ErrInvalidSender - } // Last but not least check for nonce errors if currentState.GetNonce(from) > tx.Nonce() { return ErrNonceTooLow @@ -748,14 +750,8 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A spammers := prque.New() for addr, list := range pool.pending { // Only evict transactions from high rollers - if uint64(list.Len()) > pool.config.AccountSlots { - // Skip local accounts as pools should maintain backlogs for themselves - for _, tx := range list.txs.items { - if !pool.locals.contains(tx.Hash()) { - spammers.Push(addr, float32(list.Len())) - } - break // Checking on transaction for locality is enough - } + if !pool.locals.containsAddress(addr) && uint64(list.Len()) > pool.config.AccountSlots { + spammers.Push(addr, float32(list.Len())) } } // Gradually drop transactions from offenders @@ -929,48 +925,41 @@ func (a addresssByHeartbeat) Len() int { return len(a) } func (a addresssByHeartbeat) Less(i, j int) bool { return a[i].heartbeat.Before(a[j].heartbeat) } func (a addresssByHeartbeat) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -// txSet represents a set of transaction hashes in which entries -// are automatically dropped after txSetDuration time -type txSet struct { - txMap map[common.Hash]struct{} - txOrd map[uint64]txOrdType - addPtr, delPtr uint64 +// accountSet is simply a map of addresses, and a signer, to be able +// to determine the address from a tx +type accountSet struct { + accounts map[common.Address]struct{} + signer types.Signer } -const txSetDuration = time.Hour * 2 - -// txOrdType represents an entry in the time-ordered list of transaction hashes -type txOrdType struct { - hash common.Hash - time time.Time -} - -// newTxSet creates a new transaction set -func newTxSet() *txSet { - return &txSet{ - txMap: make(map[common.Hash]struct{}), - txOrd: make(map[uint64]txOrdType), +func newAccountSet(signer types.Signer) *accountSet { + return &accountSet{ + accounts: make(map[common.Address]struct{}), + signer: signer, } } -// contains returns true if the set contains the given transaction hash -// (not thread safe, should be called from a locked environment) -func (ts *txSet) contains(hash common.Hash) bool { - _, ok := ts.txMap[hash] - return ok +// containsAddress checks if a given address is within the set +func (as *accountSet) containsAddress(address common.Address) bool { + _, exist := as.accounts[address] + return exist } -// add adds a transaction hash to the set, then removes entries older than txSetDuration -// (not thread safe, should be called from a locked environment) -func (ts *txSet) add(hash common.Hash) { - ts.txMap[hash] = struct{}{} - now := time.Now() - ts.txOrd[ts.addPtr] = txOrdType{hash: hash, time: now} - ts.addPtr++ - delBefore := now.Add(-txSetDuration) - for ts.delPtr < ts.addPtr && ts.txOrd[ts.delPtr].time.Before(delBefore) { - delete(ts.txMap, ts.txOrd[ts.delPtr].hash) - delete(ts.txOrd, ts.delPtr) - ts.delPtr++ +// contains checks if the sender of a given tx is within the set +func (as *accountSet) contains(tx *types.Transaction) bool { + if address, err := types.Sender(as.signer, tx); err == nil { + return as.containsAddress(address) } + return false +} + +// add a transaction sender to the set +// if sender can't be derived, this is a no-op (no errors returned) +func (as *accountSet) add(tx *types.Transaction) { + if address, err := types.Sender(as.signer, tx); err == nil { + if _, exist := as.accounts[address]; !exist { + as.accounts[address] = struct{}{} + } + } + } From a633a2d7ea8aadb1d435679449d002de880fab30 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 30 Jun 2017 22:55:10 +0200 Subject: [PATCH 32/43] core: Prevent local tx:s from being discarded --- core/tx_pool.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index 3f758957a7..143689c594 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -725,12 +725,14 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A pool.promoteTx(addr, hash, tx) } // Drop all transactions over the allowed limit - for _, tx := range list.Cap(int(pool.config.AccountQueue)) { - hash := tx.Hash() - delete(pool.all, hash) - pool.priced.Removed() - queuedRateLimitCounter.Inc(1) - log.Trace("Removed cap-exceeding queued transaction", "hash", hash) + if !pool.locals.containsAddress(addr) { + for _, tx := range list.Cap(int(pool.config.AccountQueue)) { + hash := tx.Hash() + delete(pool.all, hash) + pool.priced.Removed() + queuedRateLimitCounter.Inc(1) + log.Trace("Removed cap-exceeding queued transaction", "hash", hash) + } } queued += uint64(list.Len()) @@ -815,7 +817,10 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A // Sort all accounts with queued transactions by heartbeat addresses := make(addresssByHeartbeat, 0, len(pool.queue)) for addr := range pool.queue { - addresses = append(addresses, addressByHeartbeat{addr, pool.beats[addr]}) + // Don't drop locals + if !pool.locals.containsAddress(addr) { + addresses = append(addresses, addressByHeartbeat{addr, pool.beats[addr]}) + } } sort.Sort(addresses) From be8f8409bc7f0f3124b7ebd1480337e4b1942113 Mon Sep 17 00:00:00 2001 From: Nick Johnson Date: Mon, 3 Jul 2017 15:17:12 +0100 Subject: [PATCH 33/43] eth/downloader, les, light: Changes in response to review --- eth/downloader/downloader.go | 39 +++++++++++++++++-------------- eth/downloader/downloader_test.go | 2 +- eth/downloader/peer.go | 32 +++++++++++++++++++++---- les/handler.go | 15 +----------- light/lightchain.go | 15 ------------ 5 files changed, 52 insertions(+), 51 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 0861e917a1..cf8e1becf6 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -115,7 +115,7 @@ type Downloader struct { syncStatsLock sync.RWMutex // Lock protecting the sync stats fields lightchain LightChain - chain BlockChain + blockchain BlockChain // Callbacks dropPeer peerDropFn // Drops a peer for misbehaving @@ -212,7 +212,7 @@ func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, chain BlockC peers: newPeerSet(), rttEstimate: uint64(rttMaxEstimate), rttConfidence: uint64(1000000), - chain: chain, + blockchain: chain, lightchain: lightchain, dropPeer: dropPeer, headerCh: make(chan dataPack, 1), @@ -246,9 +246,9 @@ func (d *Downloader) Progress() ethereum.SyncProgress { current := uint64(0) switch d.mode { case FullSync: - current = d.chain.CurrentBlock().NumberU64() + current = d.blockchain.CurrentBlock().NumberU64() case FastSync: - current = d.chain.CurrentFastBlock().NumberU64() + current = d.blockchain.CurrentFastBlock().NumberU64() case LightSync: current = d.lightchain.CurrentHeader().Number.Uint64() } @@ -281,6 +281,11 @@ func (d *Downloader) RegisterPeer(id string, version int, peer Peer) error { return nil } +// RegisterLightPeer injects a light client peer +func (d *Downloader) RegisterLightPeer(id string, version int, peer LightPeer) error { + return d.RegisterPeer(id, version, &lightPeerWrapper{peer}) +} + // UnregisterPeer remove a peer from the known list, preventing any action from // the specified peer. An effort is also made to return any pending fetches into // the queue. @@ -597,9 +602,9 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err p.log.Debug("Looking for common ancestor", "local", ceil, "remote", height) if d.mode == FullSync { - ceil = d.chain.CurrentBlock().NumberU64() + ceil = d.blockchain.CurrentBlock().NumberU64() } else if d.mode == FastSync { - ceil = d.chain.CurrentFastBlock().NumberU64() + ceil = d.blockchain.CurrentFastBlock().NumberU64() } if ceil >= MaxForkAncestry { floor = int64(ceil - MaxForkAncestry) @@ -659,7 +664,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err continue } // Otherwise check if we already know the header or not - if (d.mode == FullSync && d.chain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash())) { + if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash())) { number, hash = headers[i].Number.Uint64(), headers[i].Hash() // If every header is known, even future ones, the peer straight out lied about its head @@ -724,7 +729,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err arrived = true // Modify the search interval based on the response - if (d.mode == FullSync && !d.chain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash())) { + if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash())) { end = check break } @@ -1147,14 +1152,14 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { } lastHeader, lastFastBlock, lastBlock := d.lightchain.CurrentHeader().Number, common.Big0, common.Big0 if d.mode != LightSync { - lastFastBlock = d.chain.CurrentFastBlock().Number() - lastBlock = d.chain.CurrentBlock().Number() + lastFastBlock = d.blockchain.CurrentFastBlock().Number() + lastBlock = d.blockchain.CurrentBlock().Number() } d.lightchain.Rollback(hashes) curFastBlock, curBlock := common.Big0, common.Big0 if d.mode != LightSync { - curFastBlock = d.chain.CurrentFastBlock().Number() - curBlock = d.chain.CurrentBlock().Number() + curFastBlock = d.blockchain.CurrentFastBlock().Number() + curBlock = d.blockchain.CurrentBlock().Number() } log.Warn("Rolled back headers", "count", len(hashes), "header", fmt.Sprintf("%d->%d", lastHeader, d.lightchain.CurrentHeader().Number), @@ -1207,7 +1212,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { // L: Request new headers up from 11 (R's TD was higher, it must have something) // R: Nothing to give if d.mode != LightSync { - if !gotHeaders && td.Cmp(d.chain.GetTdByHash(d.chain.CurrentBlock().Hash())) > 0 { + if !gotHeaders && td.Cmp(d.blockchain.GetTdByHash(d.blockchain.CurrentBlock().Hash())) > 0 { return errStallingPeer } } @@ -1345,7 +1350,7 @@ func (d *Downloader) importBlockResults(results []*fetchResult) error { for i, result := range results[:items] { blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) } - if index, err := d.chain.InsertChain(blocks); err != nil { + if index, err := d.blockchain.InsertChain(blocks); err != nil { log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) return errInvalidChain } @@ -1434,7 +1439,7 @@ func (d *Downloader) commitFastSyncData(results []*fetchResult, stateSync *state blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) receipts[i] = result.Receipts } - if index, err := d.chain.InsertReceiptChain(blocks, receipts); err != nil { + if index, err := d.blockchain.InsertReceiptChain(blocks, receipts); err != nil { log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) return errInvalidChain } @@ -1452,10 +1457,10 @@ func (d *Downloader) commitPivotBlock(result *fetchResult) error { return err } log.Debug("Committing fast sync pivot as new head", "number", b.Number(), "hash", b.Hash()) - if _, err := d.chain.InsertReceiptChain([]*types.Block{b}, []types.Receipts{result.Receipts}); err != nil { + if _, err := d.blockchain.InsertReceiptChain([]*types.Block{b}, []types.Receipts{result.Receipts}); err != nil { return err } - return d.chain.FastSyncCommitHead(b.Hash()) + return d.blockchain.FastSyncCommitHead(b.Hash()) } // DeliverHeaders injects a new batch of block headers received from a remote diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index bad2851d4a..b354682a14 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -288,7 +288,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block { return dl.genesis } -// FastSynccommitHead manually sets the head block to a given hash. +// FastSyncCommitHead manually sets the head block to a given hash. func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct if block := dl.GetBlockByHash(hash); block != nil { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 71a6926883..96870da508 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -75,19 +75,43 @@ type peerConnection struct { lock sync.RWMutex } -type Peer interface { +type LightPeer interface { Head() (common.Hash, *big.Int) RequestHeadersByHash(common.Hash, int, int, bool) error RequestHeadersByNumber(uint64, int, int, bool) error +} + +type Peer interface { + LightPeer RequestBodies([]common.Hash) error RequestReceipts([]common.Hash) error RequestNodeData([]common.Hash) error } -// newPeer create a new downloader peer, with specific hash and block retrieval -// mechanisms. -func newPeerConnection(id string, version int, peer Peer, logger log.Logger) *peerConnection { +// lightPeerWrapper wraps a LightPeer struct, stubbing out the Peer-only methods. +type lightPeerWrapper struct { + peer LightPeer +} +func (w *lightPeerWrapper) Head() (common.Hash, *big.Int) { return w.peer.Head() } +func (w *lightPeerWrapper) RequestHeadersByHash(h common.Hash, amount int, skip int, reverse bool) error { + return w.peer.RequestHeadersByHash(h, amount, skip, reverse) +} +func (w *lightPeerWrapper) RequestHeadersByNumber(i uint64, amount int, skip int, reverse bool) error { + return w.peer.RequestHeadersByNumber(i, amount, skip, reverse) +} +func (w *lightPeerWrapper) RequestBodies([]common.Hash) error { + panic("RequestBodies not supported in light client mode sync") +} +func (w *lightPeerWrapper) RequestReceipts([]common.Hash) error { + panic("RequestReceipts not supported in light client mode sync") +} +func (w *lightPeerWrapper) RequestNodeData([]common.Hash) error { + panic("RequestNodeData not supported in light client mode sync") +} + +// newPeerConnection creates a new downloader peer +func newPeerConnection(id string, version int, peer Peer, logger log.Logger) *peerConnection { return &peerConnection{ id: id, lacking: make(map[common.Hash]struct{}), diff --git a/les/handler.go b/les/handler.go index 967b315219..39045ecbea 100644 --- a/les/handler.go +++ b/les/handler.go @@ -895,26 +895,13 @@ func (pc *peerConnection) RequestHeadersByNumber(origin uint64, amount int, skip return nil } -func (pc *peerConnection) RequestBodies(hashes []common.Hash) error { - panic("RequestBodies not supported in light client mode sync") - -} -func (pc *peerConnection) RequestReceipts(hashes []common.Hash) error { - panic("RequestReceipts not supported in light client mode sync") - -} - -func (pc *peerConnection) RequestNodeData(hashes []common.Hash) error { - panic("RequestNodeData not supported in light client mode sync") -} - func (d *downloaderPeerNotify) registerPeer(p *peer) { pm := (*ProtocolManager)(d) pc := &peerConnection{ manager: pm, peer: p, } - pm.downloader.RegisterPeer(p.id, ethVersion, pc) + pm.downloader.RegisterLightPeer(p.id, ethVersion, pc) } func (d *downloaderPeerNotify) unregisterPeer(p *peer) { diff --git a/light/lightchain.go b/light/lightchain.go index ec7ca3a0c2..87436f4a57 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -389,21 +389,6 @@ func (self *LightChain) CurrentHeader() *types.Header { return self.hc.CurrentHeader() } -// CurrentBlock exists for interface compatibility and always returns nil -func (self *LightChain) CurrentBlock() *types.Block { - return nil -} - -// CurrentFastBlock exists for interface compatibility and always returns nil -func (self *LightChain) CurrentFastBlock() *types.Block { - return nil -} - -// FastSyncCommitHead exists for interface compatibility and does nothing -func (self *LightChain) FastSyncCommitHead(h common.Hash) error { - return nil -} - // GetTd retrieves a block's total difficulty in the canonical chain from the // database by hash and number, caching it if found. func (self *LightChain) GetTd(hash common.Hash, number uint64) *big.Int { From 8f12d76a4723ba011d033533871a0e06e1243d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 4 Jul 2017 12:28:58 +0300 Subject: [PATCH 34/43] params: remove redundant consts, disable metro on AllProtocolChanges --- cmd/geth/dao_test.go | 2 +- cmd/swarm/main.go | 4 ++-- core/genesis.go | 6 +++--- core/genesis_test.go | 16 +++++++-------- light/lightchain.go | 2 +- params/config.go | 28 ++++++++++++++++---------- params/dao.go | 9 --------- params/util.go | 47 ------------------------------------------- tests/state_test.go | 46 +++++++++++++++++++++--------------------- tests/vm_test_util.go | 4 ++-- 10 files changed, 57 insertions(+), 107 deletions(-) delete mode 100644 params/util.go diff --git a/cmd/geth/dao_test.go b/cmd/geth/dao_test.go index 8cc66aabfe..a8dbc51630 100644 --- a/cmd/geth/dao_test.go +++ b/cmd/geth/dao_test.go @@ -89,7 +89,7 @@ func TestDAOForkBlockNewChain(t *testing.T) { expectVote bool }{ // Test DAO Default Mainnet - {"", params.MainNetDAOForkBlock, true}, + {"", params.MainnetChainConfig.DAOForkBlock, true}, // test DAO Init Old Privnet {daoOldGenesis, nil, false}, // test DAO Default No Fork Privnet diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 6b4feb0bcc..4ae06a1c99 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -380,11 +380,11 @@ func detectEnsAddr(client *rpc.Client) (common.Address, error) { switch { - case version == "1" && block.Hash() == params.MainNetGenesisHash: + case version == "1" && block.Hash() == params.MainnetGenesisHash: log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress) return ens.MainNetAddress, nil - case version == "3" && block.Hash() == params.TestNetGenesisHash: + case version == "3" && block.Hash() == params.TestnetGenesisHash: log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress) return ens.TestNetAddress, nil diff --git a/core/genesis.go b/core/genesis.go index 5815d5901b..d587011f05 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -143,7 +143,7 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig // Special case: don't change the existing config of a non-mainnet chain if no new // config is supplied. These chains would get AllProtocolChanges (and a compat error) // if we just continued here. - if genesis == nil && stored != params.MainNetGenesisHash { + if genesis == nil && stored != params.MainnetGenesisHash { return storedcfg, stored, nil } @@ -164,9 +164,9 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { switch { case g != nil: return g.Config - case ghash == params.MainNetGenesisHash: + case ghash == params.MainnetGenesisHash: return params.MainnetChainConfig - case ghash == params.TestNetGenesisHash: + case ghash == params.TestnetGenesisHash: return params.TestnetChainConfig default: return params.AllProtocolChanges diff --git a/core/genesis_test.go b/core/genesis_test.go index 4312a80b89..bc82fe54ea 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -32,12 +32,12 @@ import ( func TestDefaultGenesisBlock(t *testing.T) { block, _ := DefaultGenesisBlock().ToBlock() - if block.Hash() != params.MainNetGenesisHash { - t.Errorf("wrong mainnet genesis hash, got %v, want %v", block.Hash(), params.MainNetGenesisHash) + if block.Hash() != params.MainnetGenesisHash { + t.Errorf("wrong mainnet genesis hash, got %v, want %v", block.Hash(), params.MainnetGenesisHash) } block, _ = DefaultTestnetGenesisBlock().ToBlock() - if block.Hash() != params.TestNetGenesisHash { - t.Errorf("wrong testnet genesis hash, got %v, want %v", block.Hash(), params.TestNetGenesisHash) + if block.Hash() != params.TestnetGenesisHash { + t.Errorf("wrong testnet genesis hash, got %v, want %v", block.Hash(), params.TestnetGenesisHash) } } @@ -73,7 +73,7 @@ func TestSetupGenesis(t *testing.T) { fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { return SetupGenesisBlock(db, nil) }, - wantHash: params.MainNetGenesisHash, + wantHash: params.MainnetGenesisHash, wantConfig: params.MainnetChainConfig, }, { @@ -82,7 +82,7 @@ func TestSetupGenesis(t *testing.T) { DefaultGenesisBlock().MustCommit(db) return SetupGenesisBlock(db, nil) }, - wantHash: params.MainNetGenesisHash, + wantHash: params.MainnetGenesisHash, wantConfig: params.MainnetChainConfig, }, { @@ -100,8 +100,8 @@ func TestSetupGenesis(t *testing.T) { customg.MustCommit(db) return SetupGenesisBlock(db, DefaultTestnetGenesisBlock()) }, - wantErr: &GenesisMismatchError{Stored: customghash, New: params.TestNetGenesisHash}, - wantHash: params.TestNetGenesisHash, + wantErr: &GenesisMismatchError{Stored: customghash, New: params.TestnetGenesisHash}, + wantHash: params.TestnetGenesisHash, wantConfig: params.TestnetChainConfig, }, { diff --git a/light/lightchain.go b/light/lightchain.go index 87436f4a57..8bbf529cce 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -94,7 +94,7 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. if bc.genesisBlock == nil { return nil, core.ErrNoGenesis } - if bc.genesisBlock.Hash() == params.MainNetGenesisHash { + if bc.genesisBlock.Hash() == params.MainnetGenesisHash { // add trusted CHT WriteTrustedCht(bc.chainDb, TrustedCht{Number: 805, Root: common.HexToHash("85e4286fe0a730390245c49de8476977afdae0eb5530b277f62a52b12313d50f")}) log.Info("Added trusted CHT for mainnet") diff --git a/params/config.go b/params/config.go index 50268f6336..f4bb6172ba 100644 --- a/params/config.go +++ b/params/config.go @@ -18,23 +18,29 @@ package params import ( "fmt" + "math" "math/big" "github.com/ethereum/go-ethereum/common" ) +var ( + MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") // Mainnet genesis hash to enforce below configs on + TestnetGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") // Testnet genesis hash to enforce below configs on +) + var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainId: MainNetChainID, - HomesteadBlock: MainNetHomesteadBlock, - DAOForkBlock: MainNetDAOForkBlock, + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(1150000), + DAOForkBlock: big.NewInt(1920000), DAOForkSupport: true, - EIP150Block: MainNetHomesteadGasRepriceBlock, - EIP150Hash: MainNetHomesteadGasRepriceHash, - EIP155Block: MainNetSpuriousDragon, - EIP158Block: MainNetSpuriousDragon, - MetropolisBlock: MainNetMetropolisBlock, + EIP150Block: big.NewInt(2463000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2675000), + EIP158Block: big.NewInt(2675000), + MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet Ethash: new(EthashConfig), } @@ -49,7 +55,7 @@ var ( EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), EIP155Block: big.NewInt(10), EIP158Block: big.NewInt(10), - MetropolisBlock: TestNetMetropolisBlock, + MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet Ethash: new(EthashConfig), } @@ -64,7 +70,7 @@ var ( EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), EIP155Block: big.NewInt(3), EIP158Block: big.NewInt(3), - MetropolisBlock: TestNetMetropolisBlock, + MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet Clique: &CliqueConfig{ Period: 15, @@ -80,7 +86,7 @@ var ( // means that all fields must be set at all times. This forces // anyone adding flags to the config to also have to set these // fields. - AllProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + AllProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(math.MaxInt64) /*disabled*/, new(EthashConfig), nil} TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) diff --git a/params/dao.go b/params/dao.go index ef8e838ff7..da3c8dfc99 100644 --- a/params/dao.go +++ b/params/dao.go @@ -22,15 +22,6 @@ import ( "github.com/ethereum/go-ethereum/common" ) -// TestNetDAOForkBlock is the block number where the DAO hard-fork commences on -// the Ethereum test network. It's enforced nil since it was decided not to do a -// testnet transition. -var TestNetDAOForkBlock *big.Int - -// MainNetDAOForkBlock is the block number where the DAO hard-fork commences on -// the Ethereum main network. -var MainNetDAOForkBlock = big.NewInt(1920000) - // DAOForkBlockExtra is the block header extra-data field to set for the DAO fork // point and a number of consecutive blocks to allow fast/light syncers to correctly // pick the side they want ("dao-hard-fork"). diff --git a/params/util.go b/params/util.go deleted file mode 100644 index d4d43d0476..0000000000 --- a/params/util.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package params - -import ( - "math" - "math/big" - - "github.com/ethereum/go-ethereum/common" -) - -var ( - TestNetGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") // Testnet genesis hash to enforce below configs on - MainNetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") // Mainnet genesis hash to enforce below configs on - - TestNetHomesteadBlock = big.NewInt(0) // Testnet homestead block - MainNetHomesteadBlock = big.NewInt(1150000) // Mainnet homestead block - - TestNetHomesteadGasRepriceBlock = big.NewInt(0) // Testnet gas reprice block - MainNetHomesteadGasRepriceBlock = big.NewInt(2463000) // Mainnet gas reprice block - - TestNetHomesteadGasRepriceHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") // Testnet gas reprice block hash (used by fast sync) - MainNetHomesteadGasRepriceHash = common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0") // Mainnet gas reprice block hash (used by fast sync) - - TestNetSpuriousDragon = big.NewInt(10) - MainNetSpuriousDragon = big.NewInt(2675000) - - TestNetMetropolisBlock = big.NewInt(math.MaxInt64) - MainNetMetropolisBlock = big.NewInt(math.MaxInt64) - - TestNetChainID = big.NewInt(3) // Test net default chain ID - MainNetChainID = big.NewInt(1) // main net default chain ID -) diff --git a/tests/state_test.go b/tests/state_test.go index 97c4e5eebf..29180942b2 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -684,7 +684,7 @@ func TestEIP158Create(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "stCreateTest.json") @@ -697,7 +697,7 @@ func TestEIP158Specific(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "stEIP158SpecificTest.json") @@ -710,7 +710,7 @@ func TestEIP158NonZeroCalls(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "stNonZeroCallsTest.json") @@ -723,7 +723,7 @@ func TestEIP158ZeroCalls(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "stZeroCallsTest.json") @@ -736,7 +736,7 @@ func TestEIP158_150Specific(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "EIP150", "stEIPSpecificTest.json") @@ -749,7 +749,7 @@ func TestEIP158_150SingleCodeGasPrice(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "EIP150", "stEIPsingleCodeGasPrices.json") @@ -762,7 +762,7 @@ func TestEIP158_150MemExpandingCalls(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "EIP150", "stMemExpandingEIPCalls.json") @@ -775,7 +775,7 @@ func TestEIP158HomesteadStateSystemOperations(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stSystemOperationsTest.json") @@ -788,7 +788,7 @@ func TestEIP158HomesteadStatePreCompiledContracts(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stPreCompiledContracts.json") @@ -801,7 +801,7 @@ func TestEIP158HomesteadStateRecursiveCreate(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stSpecialTest.json") @@ -814,7 +814,7 @@ func TestEIP158HomesteadStateRefund(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stRefundTest.json") @@ -827,7 +827,7 @@ func TestEIP158HomesteadStateInitCode(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stInitCodeTest.json") @@ -840,7 +840,7 @@ func TestEIP158HomesteadStateLog(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stLogTests.json") @@ -853,7 +853,7 @@ func TestEIP158HomesteadStateTransaction(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stTransactionTest.json") @@ -866,7 +866,7 @@ func TestEIP158HomesteadCallCreateCallCode(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stCallCreateCallCodeTest.json") @@ -879,7 +879,7 @@ func TestEIP158HomesteadCallCodes(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stCallCodes.json") @@ -892,7 +892,7 @@ func TestEIP158HomesteadMemory(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stMemoryTest.json") @@ -905,7 +905,7 @@ func TestEIP158HomesteadMemoryStress(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } if os.Getenv("TEST_VM_COMPLEX") == "" { @@ -921,7 +921,7 @@ func TestEIP158HomesteadQuadraticComplexity(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } if os.Getenv("TEST_VM_COMPLEX") == "" { @@ -937,7 +937,7 @@ func TestEIP158HomesteadWallet(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stWalletTest.json") @@ -950,7 +950,7 @@ func TestEIP158HomesteadDelegateCodes(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stCallDelegateCodes.json") @@ -963,7 +963,7 @@ func TestEIP158HomesteadDelegateCodesCallCode(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stCallDelegateCodesCallCode.json") @@ -976,7 +976,7 @@ func TestEIP158HomesteadBounds(t *testing.T) { chainConfig := ¶ms.ChainConfig{ HomesteadBlock: new(big.Int), EIP150Block: big.NewInt(2457000), - EIP158Block: params.MainNetSpuriousDragon, + EIP158Block: params.MainnetChainConfig.EIP158Block, } fn := filepath.Join(stateTestDir, "EIP158", "Homestead", "stBoundsTest.json") diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index d2ddee039e..e7fe74f49b 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -215,8 +215,8 @@ func runVmTest(test VmTest) error { func RunVm(statedb *state.StateDB, env, exec map[string]string) ([]byte, []*types.Log, *big.Int, error) { chainConfig := ¶ms.ChainConfig{ - HomesteadBlock: params.MainNetHomesteadBlock, - DAOForkBlock: params.MainNetDAOForkBlock, + HomesteadBlock: params.MainnetChainConfig.HomesteadBlock, + DAOForkBlock: params.MainnetChainConfig.DAOForkBlock, DAOForkSupport: true, } var ( From fe13949d9d938316c2c1b037b2cbce8e7bead525 Mon Sep 17 00:00:00 2001 From: Nick Johnson Date: Wed, 5 Jul 2017 10:42:37 +0100 Subject: [PATCH 35/43] eth/downloader: Doc fixes --- eth/downloader/downloader.go | 4 +++- eth/downloader/peer.go | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index cf8e1becf6..6ac58140a3 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -153,6 +153,7 @@ type Downloader struct { chainInsertHook func([]*fetchResult) // Method to call upon inserting a chain of blocks (possibly in multiple invocations) } +// LightChain encapsulates functions required to synchronise a light chain. type LightChain interface { // HasHeader verifies a header's presence in the local chain. HasHeader(common.Hash) bool @@ -173,6 +174,7 @@ type LightChain interface { Rollback([]common.Hash) } +// BlockChain encapsulates functions required to sync a (full or fast) blockchain. type BlockChain interface { LightChain @@ -281,7 +283,7 @@ func (d *Downloader) RegisterPeer(id string, version int, peer Peer) error { return nil } -// RegisterLightPeer injects a light client peer +// RegisterLightPeer injects a light client peer, wrapping it so it appears as a regular peer. func (d *Downloader) RegisterLightPeer(id string, version int, peer LightPeer) error { return d.RegisterPeer(id, version, &lightPeerWrapper{peer}) } diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 96870da508..d0dc9a8aa6 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -45,7 +45,7 @@ var ( errNotRegistered = errors.New("peer is not registered") ) -// peer represents an active peer from which hashes and blocks are retrieved. +// peerConnection represents an active peer from which hashes and blocks are retrieved. type peerConnection struct { id string // Unique identifier of the peer @@ -75,12 +75,14 @@ type peerConnection struct { lock sync.RWMutex } +// LightPeer encapsulates the methods required to synchronise with a remote light peer. type LightPeer interface { Head() (common.Hash, *big.Int) RequestHeadersByHash(common.Hash, int, int, bool) error RequestHeadersByNumber(uint64, int, int, bool) error } +// Peer encapsulates the methods required to synchronise with a remote full peer. type Peer interface { LightPeer RequestBodies([]common.Hash) error @@ -110,7 +112,7 @@ func (w *lightPeerWrapper) RequestNodeData([]common.Hash) error { panic("RequestNodeData not supported in light client mode sync") } -// newPeerConnection creates a new downloader peer +// newPeerConnection creates a new downloader peer. func newPeerConnection(id string, version int, peer Peer, logger log.Logger) *peerConnection { return &peerConnection{ id: id, From 48ee7f9de7da0455b80ee09f498dbce54127103a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 5 Jul 2017 16:51:55 +0300 Subject: [PATCH 36/43] core, eth, les: polish txpool API around local/remote txs --- core/tx_list.go | 16 ++--- core/tx_pool.go | 157 ++++++++++++++++++++++++------------------- core/tx_pool_test.go | 108 ++++++++++++++--------------- eth/api_backend.go | 3 +- eth/handler.go | 2 +- eth/helper_test.go | 4 +- eth/protocol.go | 4 +- eth/protocol_test.go | 2 +- les/handler.go | 6 +- 9 files changed, 153 insertions(+), 149 deletions(-) diff --git a/core/tx_list.go b/core/tx_list.go index e12af4a891..4593943be9 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -420,7 +420,7 @@ func (l *txPricedList) Removed() { heap.Init(l.items) } -// Discard finds all the transactions below the given price threshold, drops them +// Cap finds all the transactions below the given price threshold, drops them // from the priced list and returs them for further removal from the entire pool. func (l *txPricedList) Cap(threshold *big.Int, local *accountSet) types.Transactions { drop := make(types.Transactions, 0, 128) // Remote underpriced transactions to drop @@ -429,9 +429,7 @@ func (l *txPricedList) Cap(threshold *big.Int, local *accountSet) types.Transact for len(*l.items) > 0 { // Discard stale transactions if found during cleanup tx := heap.Pop(l.items).(*types.Transaction) - - hash := tx.Hash() - if _, ok := (*l.all)[hash]; !ok { + if _, ok := (*l.all)[tx.Hash()]; !ok { l.stales-- continue } @@ -440,7 +438,7 @@ func (l *txPricedList) Cap(threshold *big.Int, local *accountSet) types.Transact break } // Non stale transaction found, discard unless local - if local.contains(tx) { + if local.containsTx(tx) { save = append(save, tx) } else { drop = append(drop, tx) @@ -456,7 +454,7 @@ func (l *txPricedList) Cap(threshold *big.Int, local *accountSet) types.Transact // lowest priced transaction currently being tracked. func (l *txPricedList) Underpriced(tx *types.Transaction, local *accountSet) bool { // Local transactions cannot be underpriced - if local.contains(tx) { + if local.containsTx(tx) { return false } // Discard stale price points if found at the heap start @@ -487,14 +485,12 @@ func (l *txPricedList) Discard(count int, local *accountSet) types.Transactions for len(*l.items) > 0 && count > 0 { // Discard stale transactions if found during cleanup tx := heap.Pop(l.items).(*types.Transaction) - - hash := tx.Hash() - if _, ok := (*l.all)[hash]; !ok { + if _, ok := (*l.all)[tx.Hash()]; !ok { l.stales-- continue } // Non stale transaction found, discard unless local - if local.contains(tx) { + if local.containsTx(tx) { save = append(save, tx) } else { drop = append(drop, tx) diff --git a/core/tx_pool.go b/core/tx_pool.go index 143689c594..a8018d74f0 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -176,12 +176,12 @@ type TxPool struct { func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, eventMux *event.TypeMux, currentStateFn stateFn, gasLimitFn func() *big.Int) *TxPool { // Sanitize the input to ensure no vulnerable gas prices are set config = (&config).sanitize() - signer := types.NewEIP155Signer(chainconfig.ChainId) + // Create the transaction pool with its initial settings pool := &TxPool{ config: config, chainconfig: chainconfig, - signer: signer, + signer: types.NewEIP155Signer(chainconfig.ChainId), pending: make(map[common.Address]*txList), queue: make(map[common.Address]*txList), beats: make(map[common.Address]time.Time), @@ -191,10 +191,10 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, eventMux *e gasLimit: gasLimitFn, gasPrice: new(big.Int).SetUint64(config.PriceLimit), pendingState: nil, - locals: newAccountSet(signer), events: eventMux.Subscribe(ChainHeadEvent{}, RemovedTransactionEvent{}), quit: make(chan struct{}), } + pool.locals = newAccountSet(pool.signer) pool.priced = newTxPricedList(&pool.all) pool.resetState() @@ -237,7 +237,7 @@ func (pool *TxPool) eventLoop() { pool.mu.Unlock() case RemovedTransactionEvent: - pool.AddBatch(ev.Txs) + pool.addTxs(ev.Txs, false) } // Handle stats reporting ticks @@ -371,52 +371,40 @@ func (pool *TxPool) Pending() (map[common.Address]types.Transactions, error) { return pending, nil } -// SetLocal marks a transaction as local, skipping gas price -// check against local miner minimum in the future -func (pool *TxPool) SetLocal(tx *types.Transaction) { - pool.mu.Lock() - defer pool.mu.Unlock() - pool.locals.add(tx) -} - -// validateTx checks whether a transaction is valid according -// to the consensus rules. -func (pool *TxPool) validateTx(tx *types.Transaction) error { - +// validateTx checks whether a transaction is valid according to the consensus +// rules and adheres to some heuristic limits of the local node (price and size). +func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { + // Heuristic limit, reject transactions over 32KB to prevent DOS attacks + if tx.Size() > 32*1024 { + return ErrOversizedData + } + // Transactions can't be negative. This may never happen using RLP decoded + // transactions but may occur if you create a transaction using the RPC. + if tx.Value().Sign() < 0 { + return ErrNegativeValue + } + // Ensure the transaction doesn't exceed the current block limit gas. + if pool.gasLimit().Cmp(tx.Gas()) < 0 { + return ErrGasLimit + } + // Make sure the transaction is signed properly from, err := types.Sender(pool.signer, tx) if err != nil { return ErrInvalidSender } - - local := pool.locals.containsAddress(from) - // Drop transactions under our own minimal accepted gas price + // Drop non-local transactions under our own minimal accepted gas price + local = local || pool.locals.contains(from) // account may be local even if the transaction arrived from the network if !local && pool.gasPrice.Cmp(tx.GasPrice()) > 0 { return ErrUnderpriced } - + // Ensure the transaction adheres to nonce ordering currentState, err := pool.currentState() if err != nil { return err } - - // Last but not least check for nonce errors if currentState.GetNonce(from) > tx.Nonce() { return ErrNonceTooLow } - - // Check the transaction doesn't exceed the current - // block limit gas. - if pool.gasLimit().Cmp(tx.Gas()) < 0 { - return ErrGasLimit - } - - // Transactions can't be negative. This may never happen - // using RLP decoded transactions but may occur if you create - // a transaction using the RPC for example. - if tx.Value().Sign() < 0 { - return ErrNegativeValue - } - // Transactor should have enough funds to cover the costs // cost == V + GP * GL if currentState.GetBalance(from).Cmp(tx.Cost()) < 0 { @@ -426,11 +414,6 @@ func (pool *TxPool) validateTx(tx *types.Transaction) error { if tx.Gas().Cmp(intrGas) < 0 { return ErrIntrinsicGas } - - // Heuristic limit, reject transactions over 32KB to prevent DOS attacks - if tx.Size() > 32*1024 { - return ErrOversizedData - } return nil } @@ -438,7 +421,11 @@ func (pool *TxPool) validateTx(tx *types.Transaction) error { // later pending promotion and execution. If the transaction is a replacement for // an already pending or queued one, it overwrites the previous and returns this // so outer code doesn't uselessly call promote. -func (pool *TxPool) add(tx *types.Transaction) (bool, error) { +// +// If a newly added transaction is marked as local, its sending account will be +// whitelisted, preventing any associated transaction from being dropped out of +// the pool due to pricing constraints. +func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { // If the transaction is already known, discard it hash := tx.Hash() if pool.all[hash] != nil { @@ -446,7 +433,7 @@ func (pool *TxPool) add(tx *types.Transaction) (bool, error) { return false, fmt.Errorf("known transaction: %x", hash) } // If the transaction fails basic validation, discard it - if err := pool.validateTx(tx); err != nil { + if err := pool.validateTx(tx, local); err != nil { log.Trace("Discarding invalid transaction", "hash", hash, "err", err) invalidTxCounter.Inc(1) return false, err @@ -488,11 +475,14 @@ func (pool *TxPool) add(tx *types.Transaction) (bool, error) { log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) return old != nil, nil } - // New transaction isn't replacing a pending one, push into queue + // New transaction isn't replacing a pending one, push into queue and potentially mark local replace, err := pool.enqueueTx(hash, tx) if err != nil { return false, err } + if local { + pool.locals.add(from) + } log.Trace("Pooled new future transaction", "hash", hash, "from", from, "to", tx.To()) return replace, nil } @@ -560,13 +550,41 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T go pool.eventMux.Post(TxPreEvent{tx}) } -// Add queues a single transaction in the pool if it is valid. -func (pool *TxPool) Add(tx *types.Transaction) error { +// AddLocal enqueues a single transaction into the pool if it is valid, marking +// the sender as a local one in the mean time, ensuring it goes around the local +// pricing constraints. +func (pool *TxPool) AddLocal(tx *types.Transaction) error { + return pool.addTx(tx, true) +} + +// AddRemote enqueues a single transaction into the pool if it is valid. If the +// sender is not among the locally tracked ones, full pricing constraints will +// apply. +func (pool *TxPool) AddRemote(tx *types.Transaction) error { + return pool.addTx(tx, false) +} + +// AddLocals enqueues a batch of transactions into the pool if they are valid, +// marking the senders as a local ones in the mean time, ensuring they go around +// the local pricing constraints. +func (pool *TxPool) AddLocals(txs []*types.Transaction) error { + return pool.addTxs(txs, true) +} + +// AddRemotes enqueues a batch of transactions into the pool if they are valid. +// If the senders are not among the locally tracked ones, full pricing constraints +// will apply. +func (pool *TxPool) AddRemotes(txs []*types.Transaction) error { + return pool.addTxs(txs, false) +} + +// addTx enqueues a single transaction into the pool if it is valid. +func (pool *TxPool) addTx(tx *types.Transaction, local bool) error { pool.mu.Lock() defer pool.mu.Unlock() // Try to inject the transaction and update any state - replace, err := pool.add(tx) + replace, err := pool.add(tx, local) if err != nil { return err } @@ -582,15 +600,15 @@ func (pool *TxPool) Add(tx *types.Transaction) error { return nil } -// AddBatch attempts to queue a batch of transactions. -func (pool *TxPool) AddBatch(txs []*types.Transaction) error { +// addTxs attempts to queue a batch of transactions if they are valid. +func (pool *TxPool) addTxs(txs []*types.Transaction, local bool) error { pool.mu.Lock() defer pool.mu.Unlock() // Add the batch of transaction, tracking the accepted ones dirty := make(map[common.Address]struct{}) for _, tx := range txs { - if replace, err := pool.add(tx); err == nil { + if replace, err := pool.add(tx, local); err == nil { if !replace { from, _ := types.Sender(pool.signer, tx) // already validated dirty[from] = struct{}{} @@ -725,7 +743,7 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A pool.promoteTx(addr, hash, tx) } // Drop all transactions over the allowed limit - if !pool.locals.containsAddress(addr) { + if !pool.locals.contains(addr) { for _, tx := range list.Cap(int(pool.config.AccountQueue)) { hash := tx.Hash() delete(pool.all, hash) @@ -752,7 +770,7 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A spammers := prque.New() for addr, list := range pool.pending { // Only evict transactions from high rollers - if !pool.locals.containsAddress(addr) && uint64(list.Len()) > pool.config.AccountSlots { + if !pool.locals.contains(addr) && uint64(list.Len()) > pool.config.AccountSlots { spammers.Push(addr, float32(list.Len())) } } @@ -818,7 +836,7 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A addresses := make(addresssByHeartbeat, 0, len(pool.queue)) for addr := range pool.queue { // Don't drop locals - if !pool.locals.containsAddress(addr) { + if !pool.locals.contains(addr) { addresses = append(addresses, addressByHeartbeat{addr, pool.beats[addr]}) } } @@ -930,13 +948,15 @@ func (a addresssByHeartbeat) Len() int { return len(a) } func (a addresssByHeartbeat) Less(i, j int) bool { return a[i].heartbeat.Before(a[j].heartbeat) } func (a addresssByHeartbeat) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -// accountSet is simply a map of addresses, and a signer, to be able -// to determine the address from a tx +// accountSet is simply a set of addresses to check for existance, and a signer +// capable of deriving addresses from transactions. type accountSet struct { accounts map[common.Address]struct{} signer types.Signer } +// newAccountSet creates a new address set with an associated signer for sender +// derivations. func newAccountSet(signer types.Signer) *accountSet { return &accountSet{ accounts: make(map[common.Address]struct{}), @@ -944,27 +964,22 @@ func newAccountSet(signer types.Signer) *accountSet { } } -// containsAddress checks if a given address is within the set -func (as *accountSet) containsAddress(address common.Address) bool { - _, exist := as.accounts[address] +// contains checks if a given address is contained within the set. +func (as *accountSet) contains(addr common.Address) bool { + _, exist := as.accounts[addr] return exist } -// contains checks if the sender of a given tx is within the set -func (as *accountSet) contains(tx *types.Transaction) bool { - if address, err := types.Sender(as.signer, tx); err == nil { - return as.containsAddress(address) +// containsTx checks if the sender of a given tx is within the set. If the sender +// cannot be derived, this method returns false. +func (as *accountSet) containsTx(tx *types.Transaction) bool { + if addr, err := types.Sender(as.signer, tx); err == nil { + return as.contains(addr) } return false } -// add a transaction sender to the set -// if sender can't be derived, this is a no-op (no errors returned) -func (as *accountSet) add(tx *types.Transaction) { - if address, err := types.Sender(as.signer, tx); err == nil { - if _, exist := as.accounts[address]; !exist { - as.accounts[address] = struct{}{} - } - } - +// add inserts a new address into the set to track. +func (as *accountSet) add(addr common.Address) { + as.accounts[addr] = struct{}{} } diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 4903bc3ca4..980805ee92 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -133,7 +133,7 @@ func TestStateChangeDuringPoolReset(t *testing.T) { t.Fatalf("Invalid nonce, want 0, got %d", nonce) } - txpool.AddBatch(types.Transactions{tx0, tx1}) + txpool.AddRemotes(types.Transactions{tx0, tx1}) nonce = txpool.State().GetNonce(address) if nonce != 2 { @@ -167,31 +167,29 @@ func TestInvalidTransactions(t *testing.T) { from, _ := deriveSender(tx) currentState, _ := pool.currentState() currentState.AddBalance(from, big.NewInt(1)) - if err := pool.Add(tx); err != ErrInsufficientFunds { + if err := pool.AddRemote(tx); err != ErrInsufficientFunds { t.Error("expected", ErrInsufficientFunds) } balance := new(big.Int).Add(tx.Value(), new(big.Int).Mul(tx.Gas(), tx.GasPrice())) currentState.AddBalance(from, balance) - if err := pool.Add(tx); err != ErrIntrinsicGas { + if err := pool.AddRemote(tx); err != ErrIntrinsicGas { t.Error("expected", ErrIntrinsicGas, "got", err) } currentState.SetNonce(from, 1) currentState.AddBalance(from, big.NewInt(0xffffffffffffff)) tx = transaction(0, big.NewInt(100000), key) - if err := pool.Add(tx); err != ErrNonceTooLow { + if err := pool.AddRemote(tx); err != ErrNonceTooLow { t.Error("expected", ErrNonceTooLow) } tx = transaction(1, big.NewInt(100000), key) pool.gasPrice = big.NewInt(1000) - if err := pool.Add(tx); err != ErrUnderpriced { + if err := pool.AddRemote(tx); err != ErrUnderpriced { t.Error("expected", ErrUnderpriced, "got", err) } - - pool.SetLocal(tx) - if err := pool.Add(tx); err != nil { + if err := pool.AddLocal(tx); err != nil { t.Error("expected", nil, "got", err) } } @@ -282,7 +280,7 @@ func TestNegativeValue(t *testing.T) { from, _ := deriveSender(tx) currentState, _ := pool.currentState() currentState.AddBalance(from, big.NewInt(1)) - if err := pool.Add(tx); err != ErrNegativeValue { + if err := pool.AddRemote(tx); err != ErrNegativeValue { t.Error("expected", ErrNegativeValue, "got", err) } } @@ -301,14 +299,14 @@ func TestTransactionChainFork(t *testing.T) { resetState() tx := transaction(0, big.NewInt(100000), key) - if _, err := pool.add(tx); err != nil { + if _, err := pool.add(tx, false); err != nil { t.Error("didn't expect error", err) } pool.RemoveBatch([]*types.Transaction{tx}) // reset the pool's internal state resetState() - if _, err := pool.add(tx); err != nil { + if _, err := pool.add(tx, false); err != nil { t.Error("didn't expect error", err) } } @@ -332,10 +330,10 @@ func TestTransactionDoubleNonce(t *testing.T) { tx3, _ := types.SignTx(types.NewTransaction(0, common.Address{}, big.NewInt(100), big.NewInt(1000000), big.NewInt(1), nil), signer, key) // Add the first two transaction, ensure higher priced stays only - if replace, err := pool.add(tx1); err != nil || replace { + if replace, err := pool.add(tx1, false); err != nil || replace { t.Errorf("first transaction insert failed (%v) or reported replacement (%v)", err, replace) } - if replace, err := pool.add(tx2); err != nil || !replace { + if replace, err := pool.add(tx2, false); err != nil || !replace { t.Errorf("second transaction insert failed (%v) or not reported replacement (%v)", err, replace) } state, _ := pool.currentState() @@ -347,7 +345,7 @@ func TestTransactionDoubleNonce(t *testing.T) { t.Errorf("transaction mismatch: have %x, want %x", tx.Hash(), tx2.Hash()) } // Add the third transaction and ensure it's not saved (smaller price) - pool.add(tx3) + pool.add(tx3, false) pool.promoteExecutables(state, []common.Address{addr}) if pool.pending[addr].Len() != 1 { t.Error("expected 1 pending transactions, got", pool.pending[addr].Len()) @@ -367,7 +365,7 @@ func TestMissingNonce(t *testing.T) { currentState, _ := pool.currentState() currentState.AddBalance(addr, big.NewInt(100000000000000)) tx := transaction(1, big.NewInt(100000), key) - if _, err := pool.add(tx); err != nil { + if _, err := pool.add(tx, false); err != nil { t.Error("didn't expect error", err) } if len(pool.pending) != 0 { @@ -390,7 +388,7 @@ func TestNonceRecovery(t *testing.T) { currentState.AddBalance(addr, big.NewInt(100000000000000)) pool.resetState() tx := transaction(n, big.NewInt(100000), key) - if err := pool.Add(tx); err != nil { + if err := pool.AddRemote(tx); err != nil { t.Error(err) } // simulate some weird re-order of transactions and missing nonce(s) @@ -598,7 +596,7 @@ func TestTransactionQueueAccountLimiting(t *testing.T) { // Keep queuing up transactions and make sure all above a limit are dropped for i := uint64(1); i <= DefaultTxPoolConfig.AccountQueue+5; i++ { - if err := pool.Add(transaction(i, big.NewInt(100000), key)); err != nil { + if err := pool.AddRemote(transaction(i, big.NewInt(100000), key)); err != nil { t.Fatalf("tx %d: failed to add transaction: %v", i, err) } if len(pool.pending) != 0 { @@ -653,7 +651,7 @@ func TestTransactionQueueGlobalLimiting(t *testing.T) { nonces[addr]++ } // Import the batch and verify that limits have been enforced - pool.AddBatch(txs) + pool.AddRemotes(txs) queued := 0 for addr, list := range pool.queue { @@ -686,7 +684,7 @@ func TestTransactionQueueTimeLimiting(t *testing.T) { // Queue up a batch of transactions for i := uint64(1); i <= DefaultTxPoolConfig.AccountQueue; i++ { - if err := pool.Add(transaction(i, big.NewInt(100000), key)); err != nil { + if err := pool.AddRemote(transaction(i, big.NewInt(100000), key)); err != nil { t.Fatalf("tx %d: failed to add transaction: %v", i, err) } } @@ -711,7 +709,7 @@ func TestTransactionPendingLimiting(t *testing.T) { // Keep queuing up transactions and make sure all above a limit are dropped for i := uint64(0); i < DefaultTxPoolConfig.AccountQueue+5; i++ { - if err := pool.Add(transaction(i, big.NewInt(100000), key)); err != nil { + if err := pool.AddRemote(transaction(i, big.NewInt(100000), key)); err != nil { t.Fatalf("tx %d: failed to add transaction: %v", i, err) } if pool.pending[account].Len() != int(i)+1 { @@ -739,7 +737,7 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { state1.AddBalance(account1, big.NewInt(1000000)) for i := uint64(0); i < DefaultTxPoolConfig.AccountQueue+5; i++ { - if err := pool1.Add(transaction(origin+i, big.NewInt(100000), key1)); err != nil { + if err := pool1.AddRemote(transaction(origin+i, big.NewInt(100000), key1)); err != nil { t.Fatalf("tx %d: failed to add transaction: %v", i, err) } } @@ -753,7 +751,7 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { for i := uint64(0); i < DefaultTxPoolConfig.AccountQueue+5; i++ { txns = append(txns, transaction(origin+i, big.NewInt(100000), key2)) } - pool2.AddBatch(txns) + pool2.AddRemotes(txns) // Ensure the batch optimization honors the same pool mechanics if len(pool1.pending) != len(pool2.pending) { @@ -808,7 +806,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { } } // Import the batch and verify that limits have been enforced - pool.AddBatch(txs) + pool.AddRemotes(txs) pending := 0 for _, list := range pool.pending { @@ -852,7 +850,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) { txs = append(txs, transaction(uint64(j), big.NewInt(100000), key)) } // Import the batch and verify that limits have been enforced - pool.AddBatch(txs) + pool.AddRemotes(txs) if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } @@ -893,7 +891,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { } } // Import the batch and verify that limits have been enforced - pool.AddBatch(txs) + pool.AddRemotes(txs) for addr, list := range pool.pending { if list.Len() != int(DefaultTxPoolConfig.AccountSlots) { @@ -937,11 +935,11 @@ func TestTransactionPoolRepricing(t *testing.T) { txs = append(txs, pricedTransaction(2, big.NewInt(100000), big.NewInt(1), keys[1])) txs = append(txs, pricedTransaction(3, big.NewInt(100000), big.NewInt(2), keys[1])) - txs = append(txs, pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[2])) - pool.SetLocal(txs[len(txs)-1]) // prevent this one from ever being dropped + ltx := pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[2]) // Import the batch and that both pending and queued transactions match up - pool.AddBatch(txs) + pool.AddRemotes(txs) + pool.AddLocal(ltx) pending, queued := pool.stats() if pending != 4 { @@ -967,10 +965,10 @@ func TestTransactionPoolRepricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // Check that we can't add the old transactions back - if err := pool.Add(pricedTransaction(1, big.NewInt(100000), big.NewInt(1), keys[0])); err != ErrUnderpriced { + if err := pool.AddRemote(pricedTransaction(1, big.NewInt(100000), big.NewInt(1), keys[0])); err != ErrUnderpriced { t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } - if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(1), keys[1])); err != ErrUnderpriced { + if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(1), keys[1])); err != ErrUnderpriced { t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } if err := validateTxPoolInternals(pool); err != nil { @@ -978,9 +976,7 @@ func TestTransactionPoolRepricing(t *testing.T) { } // However we can add local underpriced transactions tx := pricedTransaction(1, big.NewInt(100000), big.NewInt(1), keys[2]) - - pool.SetLocal(tx) // prevent this one from ever being dropped - if err := pool.Add(tx); err != nil { + if err := pool.AddLocal(tx); err != nil { t.Fatalf("failed to add underpriced local transaction: %v", err) } if pending, _ = pool.stats(); pending != 3 { @@ -1027,11 +1023,11 @@ func TestTransactionPoolUnderpricing(t *testing.T) { txs = append(txs, pricedTransaction(1, big.NewInt(100000), big.NewInt(1), keys[1])) - txs = append(txs, pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[2])) - pool.SetLocal(txs[len(txs)-1]) // prevent this one from ever being dropped + ltx := pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[2]) // Import the batch and that both pending and queued transactions match up - pool.AddBatch(txs) + pool.AddRemotes(txs) + pool.AddLocal(ltx) pending, queued := pool.stats() if pending != 3 { @@ -1044,17 +1040,17 @@ func TestTransactionPoolUnderpricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // Ensure that adding an underpriced transaction on block limit fails - if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[1])); err != ErrUnderpriced { + if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(1), keys[1])); err != ErrUnderpriced { t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } // Ensure that adding high priced transactions drops cheap ones, but not own - if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(3), keys[1])); err != nil { + if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(3), keys[1])); err != nil { t.Fatalf("failed to add well priced transaction: %v", err) } - if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(4), keys[1])); err != nil { + if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(4), keys[1])); err != nil { t.Fatalf("failed to add well priced transaction: %v", err) } - if err := pool.Add(pricedTransaction(3, big.NewInt(100000), big.NewInt(5), keys[1])); err != nil { + if err := pool.AddRemote(pricedTransaction(3, big.NewInt(100000), big.NewInt(5), keys[1])); err != nil { t.Fatalf("failed to add well priced transaction: %v", err) } pending, queued = pool.stats() @@ -1069,9 +1065,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { } // Ensure that adding local transactions can push out even higher priced ones tx := pricedTransaction(1, big.NewInt(100000), big.NewInt(0), keys[2]) - - pool.SetLocal(tx) // prevent this one from ever being dropped - if err := pool.Add(tx); err != nil { + if err := pool.AddLocal(tx); err != nil { t.Fatalf("failed to add underpriced local transaction: %v", err) } pending, queued = pool.stats() @@ -1106,43 +1100,43 @@ func TestTransactionReplacement(t *testing.T) { price := int64(100) threshold := (price * (100 + int64(DefaultTxPoolConfig.PriceBump))) / 100 - if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(1), key)); err != nil { + if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(1), key)); err != nil { t.Fatalf("failed to add original cheap pending transaction: %v", err) } - if err := pool.Add(pricedTransaction(0, big.NewInt(100001), big.NewInt(1), key)); err != ErrReplaceUnderpriced { + if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100001), big.NewInt(1), key)); err != ErrReplaceUnderpriced { t.Fatalf("original cheap pending transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) } - if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(2), key)); err != nil { + if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(2), key)); err != nil { t.Fatalf("failed to replace original cheap pending transaction: %v", err) } - if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(price), key)); err != nil { + if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(price), key)); err != nil { t.Fatalf("failed to add original proper pending transaction: %v", err) } - if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(threshold), key)); err != ErrReplaceUnderpriced { + if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(threshold), key)); err != ErrReplaceUnderpriced { t.Fatalf("original proper pending transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) } - if err := pool.Add(pricedTransaction(0, big.NewInt(100000), big.NewInt(threshold+1), key)); err != nil { + if err := pool.AddRemote(pricedTransaction(0, big.NewInt(100000), big.NewInt(threshold+1), key)); err != nil { t.Fatalf("failed to replace original proper pending transaction: %v", err) } // Add queued transactions, ensuring the minimum price bump is enforced for replacement (for ultra low prices too) - if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(1), key)); err != nil { + if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(1), key)); err != nil { t.Fatalf("failed to add original queued transaction: %v", err) } - if err := pool.Add(pricedTransaction(2, big.NewInt(100001), big.NewInt(1), key)); err != ErrReplaceUnderpriced { + if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100001), big.NewInt(1), key)); err != ErrReplaceUnderpriced { t.Fatalf("original queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) } - if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(2), key)); err != nil { + if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(2), key)); err != nil { t.Fatalf("failed to replace original queued transaction: %v", err) } - if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(price), key)); err != nil { + if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(price), key)); err != nil { t.Fatalf("failed to add original queued transaction: %v", err) } - if err := pool.Add(pricedTransaction(2, big.NewInt(100001), big.NewInt(threshold), key)); err != ErrReplaceUnderpriced { + if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100001), big.NewInt(threshold), key)); err != ErrReplaceUnderpriced { t.Fatalf("original queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) } - if err := pool.Add(pricedTransaction(2, big.NewInt(100000), big.NewInt(threshold+1), key)); err != nil { + if err := pool.AddRemote(pricedTransaction(2, big.NewInt(100000), big.NewInt(threshold+1), key)); err != nil { t.Fatalf("failed to replace original queued transaction: %v", err) } if err := validateTxPoolInternals(pool); err != nil { @@ -1213,7 +1207,7 @@ func BenchmarkPoolInsert(b *testing.B) { // Benchmark importing the transactions into the queue b.ResetTimer() for _, tx := range txs { - pool.Add(tx) + pool.AddRemote(tx) } } @@ -1239,6 +1233,6 @@ func benchmarkPoolBatchInsert(b *testing.B, size int) { // Benchmark importing the transactions into the queue b.ResetTimer() for _, batch := range batches { - pool.AddBatch(batch) + pool.AddRemotes(batch) } } diff --git a/eth/api_backend.go b/eth/api_backend.go index 166b5084d3..a60b36e65a 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -119,8 +119,7 @@ func (b *EthApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) b.eth.txMu.Lock() defer b.eth.txMu.Unlock() - b.eth.txPool.SetLocal(signedTx) - return b.eth.txPool.Add(signedTx) + return b.eth.txPool.AddLocal(signedTx) } func (b *EthApiBackend) RemoveTx(txHash common.Hash) { diff --git a/eth/handler.go b/eth/handler.go index 1af9e755ba..29f3327993 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -661,7 +661,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } p.MarkTransaction(tx.Hash()) } - pm.txpool.AddBatch(txs) + pm.txpool.AddRemotes(txs) default: return errResp(ErrInvalidMsgCode, "%v", msg.Code) diff --git a/eth/helper_test.go b/eth/helper_test.go index 0260b9d77c..546478a3ee 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -94,9 +94,9 @@ type testTxPool struct { lock sync.RWMutex // Protects the transaction pool } -// AddBatch appends a batch of transactions to the pool, and notifies any +// AddRemotes appends a batch of transactions to the pool, and notifies any // listeners if the addition channel is non nil -func (p *testTxPool) AddBatch(txs []*types.Transaction) error { +func (p *testTxPool) AddRemotes(txs []*types.Transaction) error { p.lock.Lock() defer p.lock.Unlock() diff --git a/eth/protocol.go b/eth/protocol.go index 4bc8bee729..376e4663e3 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -94,8 +94,8 @@ var errorToString = map[int]string{ } type txPool interface { - // AddBatch should add the given transactions to the pool. - AddBatch([]*types.Transaction) error + // AddRemotes should add the given transactions to the pool. + AddRemotes([]*types.Transaction) error // Pending should return pending transactions. // The slice should be modifiable by the caller. diff --git a/eth/protocol_test.go b/eth/protocol_test.go index 2056ee0a8e..d3a44ae91f 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -130,7 +130,7 @@ func testSendTransactions(t *testing.T, protocol int) { for nonce := range alltxs { alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), txsize) } - pm.txpool.AddBatch(alltxs) + pm.txpool.AddRemotes(alltxs) // Connect several peers. They should all receive the pending transactions. var wg sync.WaitGroup diff --git a/les/handler.go b/les/handler.go index 77bc077a2e..a8b2486a66 100644 --- a/les/handler.go +++ b/les/handler.go @@ -87,8 +87,8 @@ type BlockChain interface { } type txPool interface { - // AddTransactions should add the given transactions to the pool. - AddBatch([]*types.Transaction) error + // AddRemotes should add the given transactions to the pool. + AddRemotes([]*types.Transaction) error } type ProtocolManager struct { @@ -803,7 +803,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrRequestRejected, "") } - if err := pm.txpool.AddBatch(txs); err != nil { + if err := pm.txpool.AddRemotes(txs); err != nil { return errResp(ErrUnexpectedResponse, "msg: %v", err) } From 4c1d0b164b61f0ab17c899061c04b7b48a9d20a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 5 Jul 2017 16:53:40 +0300 Subject: [PATCH 37/43] eth: drop leftover from previous nonce protection scheme --- eth/api_backend.go | 22 ---------------------- eth/backend.go | 1 - 2 files changed, 23 deletions(-) diff --git a/eth/api_backend.go b/eth/api_backend.go index a60b36e65a..7ef7c030dd 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -116,28 +116,18 @@ func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state *sta } func (b *EthApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { - b.eth.txMu.Lock() - defer b.eth.txMu.Unlock() - return b.eth.txPool.AddLocal(signedTx) } func (b *EthApiBackend) RemoveTx(txHash common.Hash) { - b.eth.txMu.Lock() - defer b.eth.txMu.Unlock() - b.eth.txPool.Remove(txHash) } func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) { - b.eth.txMu.Lock() - defer b.eth.txMu.Unlock() - pending, err := b.eth.txPool.Pending() if err != nil { return nil, err } - var txs types.Transactions for _, batch := range pending { txs = append(txs, batch...) @@ -146,30 +136,18 @@ func (b *EthApiBackend) GetPoolTransactions() (types.Transactions, error) { } func (b *EthApiBackend) GetPoolTransaction(hash common.Hash) *types.Transaction { - b.eth.txMu.Lock() - defer b.eth.txMu.Unlock() - return b.eth.txPool.Get(hash) } func (b *EthApiBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { - b.eth.txMu.Lock() - defer b.eth.txMu.Unlock() - return b.eth.txPool.State().GetNonce(addr), nil } func (b *EthApiBackend) Stats() (pending int, queued int) { - b.eth.txMu.Lock() - defer b.eth.txMu.Unlock() - return b.eth.txPool.Stats() } func (b *EthApiBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { - b.eth.txMu.Lock() - defer b.eth.txMu.Unlock() - return b.eth.TxPool().Content() } diff --git a/eth/backend.go b/eth/backend.go index 75e0e737b1..78478e86e9 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -63,7 +63,6 @@ type Ethereum struct { stopDbUpgrade func() // stop chain db sequential key upgrade // Handlers txPool *core.TxPool - txMu sync.Mutex blockchain *core.BlockChain protocolManager *ProtocolManager lesServer LesServer From 5e38f7a664aa5401117f1e1705ec97476b19411e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 5 Jul 2017 17:06:05 +0300 Subject: [PATCH 38/43] cmd, core: add --txpool.nolocals to disable local price exemptions --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 7 +++++++ core/tx_pool.go | 6 ++++-- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 7e0242d8f0..607414bbb6 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -66,6 +66,7 @@ var ( utils.EthashDatasetDirFlag, utils.EthashDatasetsInMemoryFlag, utils.EthashDatasetsOnDiskFlag, + utils.TxPoolNoLocalsFlag, utils.TxPoolPriceLimitFlag, utils.TxPoolPriceBumpFlag, utils.TxPoolAccountSlotsFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 022c8e899e..275aad674b 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -95,6 +95,7 @@ var AppHelpFlagGroups = []flagGroup{ { Name: "TRANSACTION POOL", Flags: []cli.Flag{ + utils.TxPoolNoLocalsFlag, utils.TxPoolPriceLimitFlag, utils.TxPoolPriceBumpFlag, utils.TxPoolAccountSlotsFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b5a593ab6e..0159364af7 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -209,6 +209,10 @@ var ( Value: eth.DefaultConfig.EthashDatasetsOnDisk, } // Transaction pool settings + TxPoolNoLocalsFlag = cli.BoolFlag{ + Name: "txpool.nolocals", + Usage: "Disables price exemptions for locally submitted transactions", + } TxPoolPriceLimitFlag = cli.Uint64Flag{ Name: "txpool.pricelimit", Usage: "Minimum gas price limit to enforce for acceptance into the pool", @@ -831,6 +835,9 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config) { } func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { + if ctx.GlobalIsSet(TxPoolNoLocalsFlag.Name) { + cfg.NoLocals = ctx.GlobalBool(TxPoolNoLocalsFlag.Name) + } if ctx.GlobalIsSet(TxPoolPriceLimitFlag.Name) { cfg.PriceLimit = ctx.GlobalUint64(TxPoolPriceLimitFlag.Name) } diff --git a/core/tx_pool.go b/core/tx_pool.go index a8018d74f0..093d3c5fde 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -99,6 +99,8 @@ type stateFn func() (*state.StateDB, error) // TxPoolConfig are the configuration parameters of the transaction pool. type TxPoolConfig struct { + NoLocals bool // Whether local transaction handling should be disabled + PriceLimit uint64 // Minimum gas price to enforce for acceptance into the pool PriceBump uint64 // Minimum price bump percentage to replace an already existing transaction (nonce) @@ -394,7 +396,7 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { } // Drop non-local transactions under our own minimal accepted gas price local = local || pool.locals.contains(from) // account may be local even if the transaction arrived from the network - if !local && pool.gasPrice.Cmp(tx.GasPrice()) > 0 { + if (!local || pool.config.NoLocals) && pool.gasPrice.Cmp(tx.GasPrice()) > 0 { return ErrUnderpriced } // Ensure the transaction adheres to nonce ordering @@ -480,7 +482,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { if err != nil { return false, err } - if local { + if local && !pool.config.NoLocals { pool.locals.add(from) } log.Trace("Pooled new future transaction", "hash", hash, "from", from, "to", tx.To()) From 88b4fe7d21bf03bbe01961dc49508bcf4edafb51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 5 Jul 2017 17:16:42 +0300 Subject: [PATCH 39/43] core: handle nolocals during add, exepmt locals from expiration --- core/tx_pool.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index 093d3c5fde..46b09c8af2 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -396,7 +396,7 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { } // Drop non-local transactions under our own minimal accepted gas price local = local || pool.locals.contains(from) // account may be local even if the transaction arrived from the network - if (!local || pool.config.NoLocals) && pool.gasPrice.Cmp(tx.GasPrice()) > 0 { + if !local && pool.gasPrice.Cmp(tx.GasPrice()) > 0 { return ErrUnderpriced } // Ensure the transaction adheres to nonce ordering @@ -482,7 +482,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { if err != nil { return false, err } - if local && !pool.config.NoLocals { + if local { pool.locals.add(from) } log.Trace("Pooled new future transaction", "hash", hash, "from", from, "to", tx.To()) @@ -556,7 +556,7 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T // the sender as a local one in the mean time, ensuring it goes around the local // pricing constraints. func (pool *TxPool) AddLocal(tx *types.Transaction) error { - return pool.addTx(tx, true) + return pool.addTx(tx, !pool.config.NoLocals) } // AddRemote enqueues a single transaction into the pool if it is valid. If the @@ -570,7 +570,7 @@ func (pool *TxPool) AddRemote(tx *types.Transaction) error { // marking the senders as a local ones in the mean time, ensuring they go around // the local pricing constraints. func (pool *TxPool) AddLocals(txs []*types.Transaction) error { - return pool.addTxs(txs, true) + return pool.addTxs(txs, !pool.config.NoLocals) } // AddRemotes enqueues a batch of transactions into the pool if they are valid. @@ -924,6 +924,11 @@ func (pool *TxPool) expirationLoop() { case <-evict.C: pool.mu.Lock() for addr := range pool.queue { + // Skip local transactions from the eviction mechanism + if pool.locals.contains(addr) { + continue + } + // Any non-locals old enough should be removed if time.Since(pool.beats[addr]) > pool.config.Lifetime { for _, tx := range pool.queue[addr].Flatten() { pool.removeTx(tx.Hash()) From f25486c3fba497da1010c38da653262e144f87a5 Mon Sep 17 00:00:00 2001 From: ligi Date: Thu, 6 Jul 2017 00:19:38 +0200 Subject: [PATCH 40/43] core: fix typo in error message (#14763) --- core/types/transaction_signing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index b0f3275b2a..ba4f2aa03a 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -28,7 +28,7 @@ import ( ) var ( - ErrInvalidChainId = errors.New("invalid chaid id for signer") + ErrInvalidChainId = errors.New("invalid chain id for signer") errAbstractSigner = errors.New("abstract signer") abstractSignerAddress = common.HexToAddress("ffffffffffffffffffffffffffffffffffffffff") From 34ec9913f628180d0ace740abfe1362995879c93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 6 Jul 2017 11:51:59 +0300 Subject: [PATCH 41/43] core: test locals support in txpool queue limits, fix The commit reworks the transaction pool queue limitation tests to cater for testing local accounts, also testing the nolocal flag. In addition, it also fixes a panic if local transactions exceeded the global queue allowance (no accounts left to drop from) and also fixes queue eviction to operate on all accounts, not just the one being updated. --- core/tx_list.go | 2 +- core/tx_pool.go | 14 +-- core/tx_pool_test.go | 266 ++++++++++++++++++++++++++++++------------- 3 files changed, 198 insertions(+), 84 deletions(-) diff --git a/core/tx_list.go b/core/tx_list.go index 4593943be9..0d87c20bc2 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -477,7 +477,7 @@ func (l *txPricedList) Underpriced(tx *types.Transaction, local *accountSet) boo } // Discard finds a number of most underpriced transactions, removes them from the -// priced list and returs them for further removal from the entire pool. +// priced list and returns them for further removal from the entire pool. func (l *txPricedList) Discard(count int, local *accountSet) types.Transactions { drop := make(types.Transactions, 0, count) // Remote underpriced transactions to drop save := make(types.Transactions, 0, 64) // Local underpriced transactions to keep diff --git a/core/tx_pool.go b/core/tx_pool.go index 46b09c8af2..8e2d1b31d8 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -716,7 +716,6 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A } } // Iterate over all accounts and promote any executable transactions - queued := uint64(0) for _, addr := range accounts { list := pool.queue[addr] if list == nil { @@ -754,8 +753,6 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A log.Trace("Removed cap-exceeding queued transaction", "hash", hash) } } - queued += uint64(list.Len()) - // Delete the entire queue entry if it became empty. if list.Empty() { delete(pool.queue, addr) @@ -833,19 +830,22 @@ func (pool *TxPool) promoteExecutables(state *state.StateDB, accounts []common.A pendingRateLimitCounter.Inc(int64(pendingBeforeCap - pending)) } // If we've queued more transactions than the hard limit, drop oldest ones + queued := uint64(0) + for _, list := range pool.queue { + queued += uint64(list.Len()) + } if queued > pool.config.GlobalQueue { // Sort all accounts with queued transactions by heartbeat addresses := make(addresssByHeartbeat, 0, len(pool.queue)) for addr := range pool.queue { - // Don't drop locals - if !pool.locals.contains(addr) { + if !pool.locals.contains(addr) { // don't drop locals addresses = append(addresses, addressByHeartbeat{addr, pool.beats[addr]}) } } sort.Sort(addresses) - // Drop transactions until the total is below the limit - for drop := queued - pool.config.GlobalQueue; drop > 0; { + // Drop transactions until the total is below the limit or only locals remain + for drop := queued - pool.config.GlobalQueue; drop > 0 && len(addresses) > 0; { addr := addresses[len(addresses)-1] list := pool.queue[addr.address] diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 980805ee92..03ece3886f 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -47,10 +47,10 @@ func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) key, _ := crypto.GenerateKey() - newPool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) - newPool.resetState() + pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + pool.resetState() - return newPool, key + return pool, key } // validateTxPoolInternals checks various consistency invariants within the pool. @@ -125,17 +125,18 @@ func TestStateChangeDuringPoolReset(t *testing.T) { gasLimitFunc := func() *big.Int { return big.NewInt(1000000000) } - txpool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, mux, stateFunc, gasLimitFunc) - txpool.resetState() + pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, mux, stateFunc, gasLimitFunc) + defer pool.Stop() + pool.resetState() - nonce := txpool.State().GetNonce(address) + nonce := pool.State().GetNonce(address) if nonce != 0 { t.Fatalf("Invalid nonce, want 0, got %d", nonce) } - txpool.AddRemotes(types.Transactions{tx0, tx1}) + pool.AddRemotes(types.Transactions{tx0, tx1}) - nonce = txpool.State().GetNonce(address) + nonce = pool.State().GetNonce(address) if nonce != 2 { t.Fatalf("Invalid nonce, want 2, got %d", nonce) } @@ -143,9 +144,9 @@ func TestStateChangeDuringPoolReset(t *testing.T) { // trigger state change in the background trigger = true - txpool.resetState() + pool.resetState() - pendingTx, err := txpool.Pending() + pendingTx, err := pool.Pending() if err != nil { t.Fatalf("Could not fetch pending transactions: %v", err) } @@ -154,7 +155,7 @@ func TestStateChangeDuringPoolReset(t *testing.T) { t.Logf("%0x: %d\n", addr, len(txs)) } - nonce = txpool.State().GetNonce(address) + nonce = pool.State().GetNonce(address) if nonce != 2 { t.Fatalf("Invalid nonce, want 2, got %d", nonce) } @@ -162,6 +163,7 @@ func TestStateChangeDuringPoolReset(t *testing.T) { func TestInvalidTransactions(t *testing.T) { pool, key := setupTxPool() + defer pool.Stop() tx := transaction(0, big.NewInt(100), key) from, _ := deriveSender(tx) @@ -196,6 +198,8 @@ func TestInvalidTransactions(t *testing.T) { func TestTransactionQueue(t *testing.T) { pool, key := setupTxPool() + defer pool.Stop() + tx := transaction(0, big.NewInt(100), key) from, _ := deriveSender(tx) currentState, _ := pool.currentState() @@ -246,6 +250,8 @@ func TestTransactionQueue(t *testing.T) { func TestRemoveTx(t *testing.T) { pool, key := setupTxPool() + defer pool.Stop() + addr := crypto.PubkeyToAddress(key.PublicKey) currentState, _ := pool.currentState() currentState.AddBalance(addr, big.NewInt(1)) @@ -275,6 +281,7 @@ func TestRemoveTx(t *testing.T) { func TestNegativeValue(t *testing.T) { pool, key := setupTxPool() + defer pool.Stop() tx, _ := types.SignTx(types.NewTransaction(0, common.Address{}, big.NewInt(-1), big.NewInt(100), big.NewInt(1), nil), types.HomesteadSigner{}, key) from, _ := deriveSender(tx) @@ -287,6 +294,8 @@ func TestNegativeValue(t *testing.T) { func TestTransactionChainFork(t *testing.T) { pool, key := setupTxPool() + defer pool.Stop() + addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { db, _ := ethdb.NewMemDatabase() @@ -313,6 +322,8 @@ func TestTransactionChainFork(t *testing.T) { func TestTransactionDoubleNonce(t *testing.T) { pool, key := setupTxPool() + defer pool.Stop() + addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { db, _ := ethdb.NewMemDatabase() @@ -361,6 +372,8 @@ func TestTransactionDoubleNonce(t *testing.T) { func TestMissingNonce(t *testing.T) { pool, key := setupTxPool() + defer pool.Stop() + addr := crypto.PubkeyToAddress(key.PublicKey) currentState, _ := pool.currentState() currentState.AddBalance(addr, big.NewInt(100000000000000)) @@ -382,6 +395,8 @@ func TestMissingNonce(t *testing.T) { func TestNonceRecovery(t *testing.T) { const n = 10 pool, key := setupTxPool() + defer pool.Stop() + addr := crypto.PubkeyToAddress(key.PublicKey) currentState, _ := pool.currentState() currentState.SetNonce(addr, n) @@ -401,6 +416,8 @@ func TestNonceRecovery(t *testing.T) { func TestRemovedTxEvent(t *testing.T) { pool, key := setupTxPool() + defer pool.Stop() + tx := transaction(0, big.NewInt(1000000), key) from, _ := deriveSender(tx) currentState, _ := pool.currentState() @@ -421,6 +438,8 @@ func TestRemovedTxEvent(t *testing.T) { func TestTransactionDropping(t *testing.T) { // Create a test account and fund it pool, key := setupTxPool() + defer pool.Stop() + account, _ := deriveSender(transaction(0, big.NewInt(0), key)) state, _ := pool.currentState() @@ -514,6 +533,8 @@ func TestTransactionDropping(t *testing.T) { func TestTransactionPostponing(t *testing.T) { // Create a test account and fund it pool, key := setupTxPool() + defer pool.Stop() + account, _ := deriveSender(transaction(0, big.NewInt(0), key)) state, _ := pool.currentState() @@ -588,6 +609,8 @@ func TestTransactionPostponing(t *testing.T) { func TestTransactionQueueAccountLimiting(t *testing.T) { // Create a test account and fund it pool, key := setupTxPool() + defer pool.Stop() + account, _ := deriveSender(transaction(0, big.NewInt(0), key)) state, _ := pool.currentState() @@ -619,19 +642,30 @@ func TestTransactionQueueAccountLimiting(t *testing.T) { // Tests that if the transaction count belonging to multiple accounts go above // some threshold, the higher transactions are dropped to prevent DOS attacks. +// +// This logic should not hold for local transactions, unless the local tracking +// mechanism is disabled. func TestTransactionQueueGlobalLimiting(t *testing.T) { - // Reduce the queue limits to shorten test time - defer func(old uint64) { DefaultTxPoolConfig.GlobalQueue = old }(DefaultTxPoolConfig.GlobalQueue) - DefaultTxPoolConfig.GlobalQueue = DefaultTxPoolConfig.AccountQueue * 3 + testTransactionQueueGlobalLimiting(t, false) +} +func TestTransactionQueueGlobalLimitingNoLocals(t *testing.T) { + testTransactionQueueGlobalLimiting(t, true) +} +func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + config := DefaultTxPoolConfig + config.NoLocals = nolocals + config.GlobalQueue = config.AccountQueue*3 - 1 // reduce the queue limits to shorten test time (-1 to make it non divisible) + + pool := NewTxPool(config, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + defer pool.Stop() pool.resetState() - // Create a number of test accounts and fund them + // Create a number of test accounts and fund them (last one will be the local) state, _ := pool.currentState() keys := make([]*ecdsa.PrivateKey, 5) @@ -639,12 +673,14 @@ func TestTransactionQueueGlobalLimiting(t *testing.T) { keys[i], _ = crypto.GenerateKey() state.AddBalance(crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } + local := keys[len(keys)-1] + // Generate and queue a batch of transactions nonces := make(map[common.Address]uint64) - txs := make(types.Transactions, 0, 3*DefaultTxPoolConfig.GlobalQueue) + txs := make(types.Transactions, 0, 3*config.GlobalQueue) for len(txs) < cap(txs) { - key := keys[rand.Intn(len(keys))] + key := keys[rand.Intn(len(keys)-1)] // skip adding transactions with the local account addr := crypto.PubkeyToAddress(key.PublicKey) txs = append(txs, transaction(nonces[addr]+1, big.NewInt(100000), key)) @@ -655,43 +691,114 @@ func TestTransactionQueueGlobalLimiting(t *testing.T) { queued := 0 for addr, list := range pool.queue { - if list.Len() > int(DefaultTxPoolConfig.AccountQueue) { - t.Errorf("addr %x: queued accounts overflown allowance: %d > %d", addr, list.Len(), DefaultTxPoolConfig.AccountQueue) + if list.Len() > int(config.AccountQueue) { + t.Errorf("addr %x: queued accounts overflown allowance: %d > %d", addr, list.Len(), config.AccountQueue) } queued += list.Len() } - if queued > int(DefaultTxPoolConfig.GlobalQueue) { - t.Fatalf("total transactions overflow allowance: %d > %d", queued, DefaultTxPoolConfig.GlobalQueue) + if queued > int(config.GlobalQueue) { + t.Fatalf("total transactions overflow allowance: %d > %d", queued, config.GlobalQueue) + } + // Generate a batch of transactions from the local account and import them + txs = txs[:0] + for i := uint64(0); i < 3*config.GlobalQueue; i++ { + txs = append(txs, transaction(i+1, big.NewInt(100000), local)) + } + pool.AddLocals(txs) + + // If locals are disabled, the previous eviction algorithm should apply here too + if nolocals { + queued := 0 + for addr, list := range pool.queue { + if list.Len() > int(config.AccountQueue) { + t.Errorf("addr %x: queued accounts overflown allowance: %d > %d", addr, list.Len(), config.AccountQueue) + } + queued += list.Len() + } + if queued > int(config.GlobalQueue) { + t.Fatalf("total transactions overflow allowance: %d > %d", queued, config.GlobalQueue) + } + } else { + // Local exemptions are enabled, make sure the local account owned the queue + if len(pool.queue) != 1 { + t.Errorf("multiple accounts in queue: have %v, want %v", len(pool.queue), 1) + } + // Also ensure no local transactions are ever dropped, even if above global limits + if queued := pool.queue[crypto.PubkeyToAddress(local.PublicKey)].Len(); uint64(queued) != 3*config.GlobalQueue { + t.Fatalf("local account queued transaction count mismatch: have %v, want %v", queued, 3*config.GlobalQueue) + } } } // Tests that if an account remains idle for a prolonged amount of time, any // non-executable transactions queued up are dropped to prevent wasting resources // on shuffling them around. -func TestTransactionQueueTimeLimiting(t *testing.T) { - // Reduce the queue limits to shorten test time - defer func(old time.Duration) { DefaultTxPoolConfig.Lifetime = old }(DefaultTxPoolConfig.Lifetime) - defer func(old time.Duration) { evictionInterval = old }(evictionInterval) - DefaultTxPoolConfig.Lifetime = time.Second - evictionInterval = time.Second +// +// This logic should not hold for local transactions, unless the local tracking +// mechanism is disabled. +func TestTransactionQueueTimeLimiting(t *testing.T) { testTransactionQueueTimeLimiting(t, false) } +func TestTransactionQueueTimeLimitingNoLocals(t *testing.T) { testTransactionQueueTimeLimiting(t, true) } - // Create a test account and fund it - pool, key := setupTxPool() - account, _ := deriveSender(transaction(0, big.NewInt(0), key)) +func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { + // Reduce the eviction interval to a testable amount + defer func(old time.Duration) { evictionInterval = old }(evictionInterval) + evictionInterval = 250 * time.Millisecond + + // Create the pool to test the non-expiration enforcement + db, _ := ethdb.NewMemDatabase() + statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) + + config := DefaultTxPoolConfig + config.Lifetime = 250 * time.Millisecond + config.NoLocals = nolocals + + pool := NewTxPool(config, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + defer pool.Stop() + pool.resetState() + + // Create two test accounts to ensure remotes expire but locals do not + local, _ := crypto.GenerateKey() + remote, _ := crypto.GenerateKey() state, _ := pool.currentState() - state.AddBalance(account, big.NewInt(1000000)) + state.AddBalance(crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) + state.AddBalance(crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) - // Queue up a batch of transactions - for i := uint64(1); i <= DefaultTxPoolConfig.AccountQueue; i++ { - if err := pool.AddRemote(transaction(i, big.NewInt(100000), key)); err != nil { - t.Fatalf("tx %d: failed to add transaction: %v", i, err) + // Add the two transactions and ensure they both are queued up + if err := pool.AddLocal(pricedTransaction(1, big.NewInt(100000), big.NewInt(1), local)); err != nil { + t.Fatalf("failed to add local transaction: %v", err) + } + if err := pool.AddRemote(pricedTransaction(1, big.NewInt(100000), big.NewInt(1), remote)); err != nil { + t.Fatalf("failed to add remote transaction: %v", err) + } + pending, queued := pool.stats() + if pending != 0 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 0) + } + if queued != 2 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) + } + if err := validateTxPoolInternals(pool); err != nil { + t.Fatalf("pool internal state corrupted: %v", err) + } + // Wait a bit for eviction to run and clean up any leftovers, and ensure only the local remains + time.Sleep(2 * config.Lifetime) + + pending, queued = pool.stats() + if pending != 0 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 0) + } + if nolocals { + if queued != 0 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) + } + } else { + if queued != 1 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) } } - // Wait until at least two expiration cycles hit and make sure the transactions are gone - time.Sleep(2 * evictionInterval) - if len(pool.queue) > 0 { - t.Fatalf("old transactions remained after eviction") + if err := validateTxPoolInternals(pool); err != nil { + t.Fatalf("pool internal state corrupted: %v", err) } } @@ -701,6 +808,8 @@ func TestTransactionQueueTimeLimiting(t *testing.T) { func TestTransactionPendingLimiting(t *testing.T) { // Create a test account and fund it pool, key := setupTxPool() + defer pool.Stop() + account, _ := deriveSender(transaction(0, big.NewInt(0), key)) state, _ := pool.currentState() @@ -775,15 +884,15 @@ func testTransactionLimitingEquivalency(t *testing.T, origin uint64) { // some hard threshold, the higher transactions are dropped to prevent DOS // attacks. func TestTransactionPendingGlobalLimiting(t *testing.T) { - // Reduce the queue limits to shorten test time - defer func(old uint64) { DefaultTxPoolConfig.GlobalSlots = old }(DefaultTxPoolConfig.GlobalSlots) - DefaultTxPoolConfig.GlobalSlots = DefaultTxPoolConfig.AccountSlots * 10 - // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + config := DefaultTxPoolConfig + config.GlobalSlots = config.AccountSlots * 10 + + pool := NewTxPool(config, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + defer pool.Stop() pool.resetState() // Create a number of test accounts and fund them @@ -800,7 +909,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { txs := types.Transactions{} for _, key := range keys { addr := crypto.PubkeyToAddress(key.PublicKey) - for j := 0; j < int(DefaultTxPoolConfig.GlobalSlots)/len(keys)*2; j++ { + for j := 0; j < int(config.GlobalSlots)/len(keys)*2; j++ { txs = append(txs, transaction(nonces[addr], big.NewInt(100000), key)) nonces[addr]++ } @@ -812,8 +921,8 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { for _, list := range pool.pending { pending += list.Len() } - if pending > int(DefaultTxPoolConfig.GlobalSlots) { - t.Fatalf("total pending transactions overflow allowance: %d > %d", pending, DefaultTxPoolConfig.GlobalSlots) + if pending > int(config.GlobalSlots) { + t.Fatalf("total pending transactions overflow allowance: %d > %d", pending, config.GlobalSlots) } if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) @@ -822,20 +931,17 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { // Tests that if transactions start being capped, transasctions are also removed from 'all' func TestTransactionCapClearsFromAll(t *testing.T) { - // Reduce the queue limits to shorten test time - defer func(old uint64) { DefaultTxPoolConfig.AccountSlots = old }(DefaultTxPoolConfig.AccountSlots) - defer func(old uint64) { DefaultTxPoolConfig.AccountQueue = old }(DefaultTxPoolConfig.AccountQueue) - defer func(old uint64) { DefaultTxPoolConfig.GlobalSlots = old }(DefaultTxPoolConfig.GlobalSlots) - - DefaultTxPoolConfig.AccountSlots = 2 - DefaultTxPoolConfig.AccountQueue = 2 - DefaultTxPoolConfig.GlobalSlots = 8 - // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + config := DefaultTxPoolConfig + config.AccountSlots = 2 + config.AccountQueue = 2 + config.GlobalSlots = 8 + + pool := NewTxPool(config, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + defer pool.Stop() pool.resetState() // Create a number of test accounts and fund them @@ -846,7 +952,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) { state.AddBalance(addr, big.NewInt(1000000)) txs := types.Transactions{} - for j := 0; j < int(DefaultTxPoolConfig.GlobalSlots)*2; j++ { + for j := 0; j < int(config.GlobalSlots)*2; j++ { txs = append(txs, transaction(uint64(j), big.NewInt(100000), key)) } // Import the batch and verify that limits have been enforced @@ -860,15 +966,15 @@ func TestTransactionCapClearsFromAll(t *testing.T) { // some hard threshold, if they are under the minimum guaranteed slot count then // the transactions are still kept. func TestTransactionPendingMinimumAllowance(t *testing.T) { - // Reduce the queue limits to shorten test time - defer func(old uint64) { DefaultTxPoolConfig.GlobalSlots = old }(DefaultTxPoolConfig.GlobalSlots) - DefaultTxPoolConfig.GlobalSlots = 0 - // Create the pool to test the limit enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + config := DefaultTxPoolConfig + config.GlobalSlots = 0 + + pool := NewTxPool(config, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + defer pool.Stop() pool.resetState() // Create a number of test accounts and fund them @@ -885,7 +991,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { txs := types.Transactions{} for _, key := range keys { addr := crypto.PubkeyToAddress(key.PublicKey) - for j := 0; j < int(DefaultTxPoolConfig.AccountSlots)*2; j++ { + for j := 0; j < int(config.AccountSlots)*2; j++ { txs = append(txs, transaction(nonces[addr], big.NewInt(100000), key)) nonces[addr]++ } @@ -894,8 +1000,8 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { pool.AddRemotes(txs) for addr, list := range pool.pending { - if list.Len() != int(DefaultTxPoolConfig.AccountSlots) { - t.Errorf("addr %x: total pending transactions mismatch: have %d, want %d", addr, list.Len(), DefaultTxPoolConfig.AccountSlots) + if list.Len() != int(config.AccountSlots) { + t.Errorf("addr %x: total pending transactions mismatch: have %d, want %d", addr, list.Len(), config.AccountSlots) } } if err := validateTxPoolInternals(pool); err != nil { @@ -914,6 +1020,7 @@ func TestTransactionPoolRepricing(t *testing.T) { statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + defer pool.Stop() pool.resetState() // Create a number of test accounts and fund them @@ -993,18 +1100,16 @@ func TestTransactionPoolRepricing(t *testing.T) { // // Note, local transactions are never allowed to be dropped. func TestTransactionPoolUnderpricing(t *testing.T) { - // Reduce the queue limits to shorten test time - defer func(old uint64) { DefaultTxPoolConfig.GlobalSlots = old }(DefaultTxPoolConfig.GlobalSlots) - DefaultTxPoolConfig.GlobalSlots = 2 - - defer func(old uint64) { DefaultTxPoolConfig.GlobalQueue = old }(DefaultTxPoolConfig.GlobalQueue) - DefaultTxPoolConfig.GlobalQueue = 2 - // Create the pool to test the pricing enforcement with db, _ := ethdb.NewMemDatabase() statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + config := DefaultTxPoolConfig + config.GlobalSlots = 2 + config.GlobalQueue = 2 + + pool := NewTxPool(config, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + defer pool.Stop() pool.resetState() // Create a number of test accounts and fund them @@ -1088,9 +1193,10 @@ func TestTransactionReplacement(t *testing.T) { statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) pool := NewTxPool(DefaultTxPoolConfig, params.TestChainConfig, new(event.TypeMux), func() (*state.StateDB, error) { return statedb, nil }, func() *big.Int { return big.NewInt(1000000) }) + defer pool.Stop() pool.resetState() - // Create a a test account to add transactions with + // Create a test account to add transactions with key, _ := crypto.GenerateKey() state, _ := pool.currentState() @@ -1153,6 +1259,8 @@ func BenchmarkPendingDemotion10000(b *testing.B) { benchmarkPendingDemotion(b, 1 func benchmarkPendingDemotion(b *testing.B, size int) { // Add a batch of transactions to a pool one by one pool, key := setupTxPool() + defer pool.Stop() + account, _ := deriveSender(transaction(0, big.NewInt(0), key)) state, _ := pool.currentState() state.AddBalance(account, big.NewInt(1000000)) @@ -1177,6 +1285,8 @@ func BenchmarkFuturePromotion10000(b *testing.B) { benchmarkFuturePromotion(b, 1 func benchmarkFuturePromotion(b *testing.B, size int) { // Add a batch of transactions to a pool one by one pool, key := setupTxPool() + defer pool.Stop() + account, _ := deriveSender(transaction(0, big.NewInt(0), key)) state, _ := pool.currentState() state.AddBalance(account, big.NewInt(1000000)) @@ -1196,6 +1306,8 @@ func benchmarkFuturePromotion(b *testing.B, size int) { func BenchmarkPoolInsert(b *testing.B) { // Generate a batch of transactions to enqueue into the pool pool, key := setupTxPool() + defer pool.Stop() + account, _ := deriveSender(transaction(0, big.NewInt(0), key)) state, _ := pool.currentState() state.AddBalance(account, big.NewInt(1000000)) @@ -1219,6 +1331,8 @@ func BenchmarkPoolBatchInsert10000(b *testing.B) { benchmarkPoolBatchInsert(b, 1 func benchmarkPoolBatchInsert(b *testing.B, size int) { // Generate a batch of transactions to enqueue into the pool pool, key := setupTxPool() + defer pool.Stop() + account, _ := deriveSender(transaction(0, big.NewInt(0), key)) state, _ := pool.currentState() state.AddBalance(account, big.NewInt(1000000)) From 65f0e905ddda9be97368839b67be5754b33a0eea Mon Sep 17 00:00:00 2001 From: Emin Mahrt Date: Fri, 7 Jul 2017 16:41:26 +0200 Subject: [PATCH 42/43] README: add missing full stop (#14766) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8d4f37f5ea..0ac9fe8fb5 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The go-ethereum project comes with several wrappers/executables found in the `cm | Command | Description | |:----------:|-------------| -| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default) archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options | +| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default) archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. | | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. | | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | | `disasm` | Bytecode disassembler to convert EVM (Ethereum Virtual Machine) bytecode into more user friendly assembly-like opcodes (e.g. `echo "6001" | disasm`). For details on the individual opcodes, please see pages 22-30 of the [Ethereum Yellow Paper](http://gavwood.com/paper.pdf). | From 09587706253a26dc58a81912419ed48672c6ec68 Mon Sep 17 00:00:00 2001 From: Daniel Sloof Date: Mon, 10 Jul 2017 12:47:50 +0200 Subject: [PATCH 43/43] internal/web3ext: fix debug.traceBlockFromFile wrapper (#14774) As stated in the documentation, this method should be called traceBlockFromFile and not traceBlockByFile. Previously this would result in a 'The method ... does not exist/is not available' error. --- internal/web3ext/web3ext.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 948d659c2b..44fabd6abb 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -208,8 +208,8 @@ web3._extend({ params: 1 }), new web3._extend.Method({ - name: 'traceBlockByFile', - call: 'debug_traceBlockByFile', + name: 'traceBlockFromFile', + call: 'debug_traceBlockFromFile', params: 1 }), new web3._extend.Method({