From da92f5b2d6482b5d0b30a2b20f397f28427a2b0d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 24 Jan 2017 20:42:47 +0100 Subject: [PATCH 01/21] core/genesis: add support for setting nonce in 'alloc' This is to be able to set `pre`-state when performing blockchain tests through Hive, we need to be able to set the nonce. --- core/genesis.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/genesis.go b/core/genesis.go index 4a440c2674..a06c404082 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -57,6 +57,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, Code string Storage map[string]string Balance string + Nonce string } } @@ -70,6 +71,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, address := common.HexToAddress(addr) statedb.AddBalance(address, common.String2Big(account.Balance)) statedb.SetCode(address, common.Hex2Bytes(account.Code)) + statedb.SetNonce(address, common.String2Big(account.Nonce).Uint64()) for key, value := range account.Storage { statedb.SetState(address, common.HexToHash(key), common.HexToHash(value)) } From 9b62facdd4bdabfed5ef98d131686c4d2606083a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Sat, 10 Dec 2016 19:02:14 +0100 Subject: [PATCH 02/21] event: deprecate TypeMux and related types The Subscription type is gone, all uses are replaced by *TypeMuxSubscription. This change is prep-work for the introduction of the new Subscription type in a later commit. gorename -from '"github.com/ethereum/go-ethereum/event"::Event' -to TypeMuxEvent gorename -from '"github.com/ethereum/go-ethereum/event"::muxsub' -to TypeMuxSubscription gofmt -w -r 'Subscription -> *TypeMuxSubscription' ./event/*.go find . -name '*.go' -and -not -regex '\./vendor/.*' \| xargs gofmt -w -r 'event.Subscription -> *event.TypeMuxSubscription' --- core/tx_pool.go | 2 +- eth/filters/filter_system.go | 4 +-- eth/handler.go | 4 +-- event/event.go | 58 +++++++++++++++--------------------- light/txpool.go | 2 +- miner/worker.go | 2 +- 6 files changed, 31 insertions(+), 41 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index 58922f12fc..ca16c1ba3c 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -89,7 +89,7 @@ type TxPool struct { gasLimit func() *big.Int // The current gas limit function callback minGasPrice *big.Int eventMux *event.TypeMux - events event.Subscription + events *event.TypeMuxSubscription localTx *txSet signer types.Signer mu sync.RWMutex diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index e0ee2ff514..3adf8111ad 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -74,7 +74,7 @@ type subscription struct { // subscription which match the subscription criteria. type EventSystem struct { mux *event.TypeMux - sub event.Subscription + sub *event.TypeMuxSubscription backend Backend lightMode bool lastHead *types.Header @@ -277,7 +277,7 @@ func (es *EventSystem) SubscribePendingTxEvents(hashes chan common.Hash) *Subscr type filterIndex map[Type]map[rpc.ID]*subscription // broadcast event to filters that match criteria. -func (es *EventSystem) broadcast(filters filterIndex, ev *event.Event) { +func (es *EventSystem) broadcast(filters filterIndex, ev *event.TypeMuxEvent) { if ev == nil { return } diff --git a/eth/handler.go b/eth/handler.go index 63ba0821fe..691fc06778 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -78,8 +78,8 @@ type ProtocolManager struct { SubProtocols []p2p.Protocol eventMux *event.TypeMux - txSub event.Subscription - minedBlockSub event.Subscription + txSub *event.TypeMuxSubscription + minedBlockSub *event.TypeMuxSubscription // channels for fetcher, syncer, txsyncLoop newPeerCh chan *peer diff --git a/event/event.go b/event/event.go index fd0bcfbd48..f8a2eb013e 100644 --- a/event/event.go +++ b/event/event.go @@ -25,33 +25,22 @@ import ( "time" ) -// Event is a time-tagged notification pushed to subscribers. -type Event struct { +// TypeMuxEvent is a time-tagged notification pushed to subscribers. +type TypeMuxEvent struct { Time time.Time Data interface{} } -// Subscription is implemented by event subscriptions. -type Subscription interface { - // Chan returns a channel that carries events. - // Implementations should return the same channel - // for any subsequent calls to Chan. - Chan() <-chan *Event - - // Unsubscribe stops delivery of events to a subscription. - // The event channel is closed. - // Unsubscribe can be called more than once. - Unsubscribe() -} - // A TypeMux dispatches events to registered receivers. Receivers can be // registered to handle events of certain type. Any operation // called after mux is stopped will return ErrMuxClosed. // // The zero value is ready to use. +// +// Deprecated: use Feed type TypeMux struct { mutex sync.RWMutex - subm map[reflect.Type][]*muxsub + subm map[reflect.Type][]*TypeMuxSubscription stopped bool } @@ -61,7 +50,7 @@ var ErrMuxClosed = errors.New("event: mux closed") // Subscribe creates a subscription for events of the given types. The // subscription's channel is closed when it is unsubscribed // or the mux is closed. -func (mux *TypeMux) Subscribe(types ...interface{}) Subscription { +func (mux *TypeMux) Subscribe(types ...interface{}) *TypeMuxSubscription { sub := newsub(mux) mux.mutex.Lock() defer mux.mutex.Unlock() @@ -72,7 +61,7 @@ func (mux *TypeMux) Subscribe(types ...interface{}) Subscription { close(sub.postC) } else { if mux.subm == nil { - mux.subm = make(map[reflect.Type][]*muxsub) + mux.subm = make(map[reflect.Type][]*TypeMuxSubscription) } for _, t := range types { rtyp := reflect.TypeOf(t) @@ -80,7 +69,7 @@ func (mux *TypeMux) Subscribe(types ...interface{}) Subscription { if find(oldsubs, sub) != -1 { panic(fmt.Sprintf("event: duplicate type %s in Subscribe", rtyp)) } - subs := make([]*muxsub, len(oldsubs)+1) + subs := make([]*TypeMuxSubscription, len(oldsubs)+1) copy(subs, oldsubs) subs[len(oldsubs)] = sub mux.subm[rtyp] = subs @@ -92,7 +81,7 @@ func (mux *TypeMux) Subscribe(types ...interface{}) Subscription { // Post sends an event to all receivers registered for the given type. // It returns ErrMuxClosed if the mux has been stopped. func (mux *TypeMux) Post(ev interface{}) error { - event := &Event{ + event := &TypeMuxEvent{ Time: time.Now(), Data: ev, } @@ -125,7 +114,7 @@ func (mux *TypeMux) Stop() { mux.mutex.Unlock() } -func (mux *TypeMux) del(s *muxsub) { +func (mux *TypeMux) del(s *TypeMuxSubscription) { mux.mutex.Lock() for typ, subs := range mux.subm { if pos := find(subs, s); pos >= 0 { @@ -139,7 +128,7 @@ func (mux *TypeMux) del(s *muxsub) { s.mux.mutex.Unlock() } -func find(slice []*muxsub, item *muxsub) int { +func find(slice []*TypeMuxSubscription, item *TypeMuxSubscription) int { for i, v := range slice { if v == item { return i @@ -148,14 +137,15 @@ func find(slice []*muxsub, item *muxsub) int { return -1 } -func posdelete(slice []*muxsub, pos int) []*muxsub { - news := make([]*muxsub, len(slice)-1) +func posdelete(slice []*TypeMuxSubscription, pos int) []*TypeMuxSubscription { + news := make([]*TypeMuxSubscription, len(slice)-1) copy(news[:pos], slice[:pos]) copy(news[pos:], slice[pos+1:]) return news } -type muxsub struct { +// TypeMuxSubscription is a subscription established through TypeMux. +type TypeMuxSubscription struct { mux *TypeMux created time.Time closeMu sync.Mutex @@ -166,13 +156,13 @@ type muxsub struct { // postC can be set to nil without affecting the return value of // Chan. postMu sync.RWMutex - readC <-chan *Event - postC chan<- *Event + readC <-chan *TypeMuxEvent + postC chan<- *TypeMuxEvent } -func newsub(mux *TypeMux) *muxsub { - c := make(chan *Event) - return &muxsub{ +func newsub(mux *TypeMux) *TypeMuxSubscription { + c := make(chan *TypeMuxEvent) + return &TypeMuxSubscription{ mux: mux, created: time.Now(), readC: c, @@ -181,16 +171,16 @@ func newsub(mux *TypeMux) *muxsub { } } -func (s *muxsub) Chan() <-chan *Event { +func (s *TypeMuxSubscription) Chan() <-chan *TypeMuxEvent { return s.readC } -func (s *muxsub) Unsubscribe() { +func (s *TypeMuxSubscription) Unsubscribe() { s.mux.del(s) s.closewait() } -func (s *muxsub) closewait() { +func (s *TypeMuxSubscription) closewait() { s.closeMu.Lock() defer s.closeMu.Unlock() if s.closed { @@ -205,7 +195,7 @@ func (s *muxsub) closewait() { s.postMu.Unlock() } -func (s *muxsub) deliver(event *Event) { +func (s *TypeMuxSubscription) deliver(event *TypeMuxEvent) { // Short circuit delivery if stale event if s.created.After(event.Time) { return diff --git a/light/txpool.go b/light/txpool.go index d0781593b0..bcdb6123de 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -47,7 +47,7 @@ type TxPool struct { signer types.Signer quit chan bool eventMux *event.TypeMux - events event.Subscription + events *event.TypeMuxSubscription mu sync.RWMutex chain *LightChain odr OdrBackend diff --git a/miner/worker.go b/miner/worker.go index 77e4e02059..49ac602537 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -90,7 +90,7 @@ type worker struct { // update loop mux *event.TypeMux - events event.Subscription + events *event.TypeMuxSubscription wg sync.WaitGroup agents map[Agent]struct{} From 6d5e100d0dc6fc0b905610850a75b5d4fa907739 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 19 Jul 2016 01:39:12 +0200 Subject: [PATCH 03/21] event: add new Subscription type and related utilities This commit introduces a new Subscription type, which is synonymous with ethereum.Subscription. It also adds a couple of utilities that make working with Subscriptions easier. The mot complex utility is Feed, a synchronisation device that implements broadcast subscriptions. Feed is slightly faster than TypeMux and will replace uses of TypeMux across the go-ethereum codebase in the future. --- event/event.go | 2 +- event/event_test.go | 30 +++- event/example_feed_test.go | 73 ++++++++ event/example_scope_test.go | 128 ++++++++++++++ event/example_subscription_test.go | 56 ++++++ event/feed.go | 240 +++++++++++++++++++++++++ event/feed_test.go | 226 ++++++++++++++++++++++++ event/subscription.go | 275 +++++++++++++++++++++++++++++ event/subscription_test.go | 121 +++++++++++++ 9 files changed, 1144 insertions(+), 7 deletions(-) create mode 100644 event/example_feed_test.go create mode 100644 event/example_scope_test.go create mode 100644 event/example_subscription_test.go create mode 100644 event/feed.go create mode 100644 event/feed_test.go create mode 100644 event/subscription.go create mode 100644 event/subscription_test.go diff --git a/event/event.go b/event/event.go index f8a2eb013e..d3e84f0f7f 100644 --- a/event/event.go +++ b/event/event.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package event implements an event multiplexer. +// Package event deals with subscriptions to real-time events. package event import ( diff --git a/event/event_test.go b/event/event_test.go index 2c56ecf29f..a12945a471 100644 --- a/event/event_test.go +++ b/event/event_test.go @@ -149,16 +149,34 @@ func emptySubscriber(mux *TypeMux, types ...interface{}) { }() } -func BenchmarkPost3(b *testing.B) { - var mux = new(TypeMux) - defer mux.Stop() - emptySubscriber(mux, testEvent(0)) - emptySubscriber(mux, testEvent(0)) - emptySubscriber(mux, testEvent(0)) +func BenchmarkPost1000(b *testing.B) { + var ( + mux = new(TypeMux) + subscribed, done sync.WaitGroup + nsubs = 1000 + ) + subscribed.Add(nsubs) + done.Add(nsubs) + for i := 0; i < nsubs; i++ { + go func() { + s := mux.Subscribe(testEvent(0)) + subscribed.Done() + for range s.Chan() { + } + done.Done() + }() + } + subscribed.Wait() + // The actual benchmark. + b.ResetTimer() for i := 0; i < b.N; i++ { mux.Post(testEvent(0)) } + + b.StopTimer() + mux.Stop() + done.Wait() } func BenchmarkPostConcurrent(b *testing.B) { diff --git a/event/example_feed_test.go b/event/example_feed_test.go new file mode 100644 index 0000000000..63436b2269 --- /dev/null +++ b/event/example_feed_test.go @@ -0,0 +1,73 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event_test + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/event" +) + +func ExampleFeed_acknowledgedEvents() { + // This example shows how the return value of Send can be used for request/reply + // interaction between event consumers and producers. + var feed event.Feed + type ackedEvent struct { + i int + ack chan<- struct{} + } + + // Consumers wait for events on the feed and acknowledge processing. + done := make(chan struct{}) + defer close(done) + for i := 0; i < 3; i++ { + ch := make(chan ackedEvent, 100) + sub := feed.Subscribe(ch) + go func() { + defer sub.Unsubscribe() + for { + select { + case ev := <-ch: + fmt.Println(ev.i) // "process" the event + ev.ack <- struct{}{} + case <-done: + return + } + } + }() + } + + // The producer sends values of type ackedEvent with increasing values of i. + // It waits for all consumers to acknowledge before sending the next event. + for i := 0; i < 3; i++ { + acksignal := make(chan struct{}) + n := feed.Send(ackedEvent{i, acksignal}) + for ack := 0; ack < n; ack++ { + <-acksignal + } + } + // Output: + // 0 + // 0 + // 0 + // 1 + // 1 + // 1 + // 2 + // 2 + // 2 +} diff --git a/event/example_scope_test.go b/event/example_scope_test.go new file mode 100644 index 0000000000..c517a8324d --- /dev/null +++ b/event/example_scope_test.go @@ -0,0 +1,128 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event_test + +import ( + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/event" +) + +// This example demonstrates how SubscriptionScope can be used to control the lifetime of +// subscriptions. +// +// Our example program consists of two servers, each of which performs a calculation when +// requested. The servers also allow subscribing to results of all computations. +type divServer struct{ results event.Feed } +type mulServer struct{ results event.Feed } + +func (s *divServer) do(a, b int) int { + r := a / b + s.results.Send(r) + return r +} + +func (s *mulServer) do(a, b int) int { + r := a * b + s.results.Send(r) + return r +} + +// The servers are contained in an App. The app controls the servers and exposes them +// through its API. +type App struct { + divServer + mulServer + scope event.SubscriptionScope +} + +func (s *App) Calc(op byte, a, b int) int { + switch op { + case '/': + return s.divServer.do(a, b) + case '*': + return s.mulServer.do(a, b) + default: + panic("invalid op") + } +} + +// The app's SubscribeResults method starts sending calculation results to the given +// channel. Subscriptions created through this method are tied to the lifetime of the App +// because they are registered in the scope. +func (s *App) SubscribeResults(op byte, ch chan<- int) event.Subscription { + switch op { + case '/': + return s.scope.Track(s.divServer.results.Subscribe(ch)) + case '*': + return s.scope.Track(s.mulServer.results.Subscribe(ch)) + default: + panic("invalid op") + } +} + +// Stop stops the App, closing all subscriptions created through SubscribeResults. +func (s *App) Stop() { + s.scope.Close() +} + +func ExampleSubscriptionScope() { + // Create the app. + var ( + app App + wg sync.WaitGroup + divs = make(chan int) + muls = make(chan int) + ) + + // Run a subscriber in the background. + divsub := app.SubscribeResults('/', divs) + mulsub := app.SubscribeResults('*', muls) + wg.Add(1) + go func() { + defer wg.Done() + defer fmt.Println("subscriber exited") + defer divsub.Unsubscribe() + defer mulsub.Unsubscribe() + for { + select { + case result := <-divs: + fmt.Println("division happened:", result) + case result := <-muls: + fmt.Println("multiplication happened:", result) + case <-divsub.Err(): + return + case <-mulsub.Err(): + return + } + } + }() + + // Interact with the app. + app.Calc('/', 22, 11) + app.Calc('*', 3, 4) + + // Stop the app. This shuts down the subscriptions, causing the subscriber to exit. + app.Stop() + wg.Wait() + + // Output: + // division happened: 2 + // multiplication happened: 12 + // subscriber exited +} diff --git a/event/example_subscription_test.go b/event/example_subscription_test.go new file mode 100644 index 0000000000..de11266893 --- /dev/null +++ b/event/example_subscription_test.go @@ -0,0 +1,56 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event_test + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/event" +) + +func ExampleNewSubscription() { + // Create a subscription that sends 10 integers on ch. + ch := make(chan int) + sub := event.NewSubscription(func(quit <-chan struct{}) error { + for i := 0; i < 10; i++ { + select { + case ch <- i: + case <-quit: + fmt.Println("unsubscribed") + return nil + } + } + return nil + }) + + // This is the consumer. It reads 5 integers, then aborts the subscription. + // Note that Unsubscribe waits until the producer has shut down. + for i := range ch { + fmt.Println(i) + if i == 4 { + sub.Unsubscribe() + break + } + } + // Output: + // 0 + // 1 + // 2 + // 3 + // 4 + // unsubscribed +} diff --git a/event/feed.go b/event/feed.go new file mode 100644 index 0000000000..bd8e26321d --- /dev/null +++ b/event/feed.go @@ -0,0 +1,240 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event + +import ( + "errors" + "reflect" + "sync" +) + +var errBadChannel = errors.New("event: Subscribe argument does not have sendable channel type") + +// Feed implements one-to-many subscriptions where the carrier of events is a channel. +// Values sent to a Feed are delivered to all subscribed channels simultaneously. +// +// Feeds can only be used with a single type. The type is determined by the first Send or +// Subscribe operation. Subsequent calls to these methods panic if the type does not +// match. +// +// The zero value is ready to use. +type Feed struct { + sendLock chan struct{} // one-element buffer, empty when held + removeSub chan interface{} // interrupts Send + sendCases caseList // the active set of select cases used by Send + + // The inbox holds newly subscribed channels until they are added to sendCases. + mu sync.Mutex + inbox caseList + etype reflect.Type + closed bool +} + +type feedTypeError struct { + got, want reflect.Type + op string +} + +func (e feedTypeError) Error() string { + return "event: wrong type in " + e.op + " got " + e.got.String() + ", want " + e.want.String() +} + +func (f *Feed) init() { + if f.sendLock != nil { + return + } + f.removeSub = make(chan interface{}) + f.sendLock = make(chan struct{}, 1) + f.sendLock <- struct{}{} + f.sendCases = caseList{{Chan: reflect.ValueOf(f.removeSub), Dir: reflect.SelectRecv}} +} + +// Subscribe adds a channel to the feed. Future sends will be delivered on the channel +// until the subscription is canceled. All channels added must have the same element type. +// +// The channel should have ample buffer space to avoid blocking other subscribers. +func (f *Feed) Subscribe(channel interface{}) Subscription { + chanval := reflect.ValueOf(channel) + chantyp := chanval.Type() + if chantyp.Kind() != reflect.Chan || chantyp.ChanDir()&reflect.SendDir == 0 { + panic(errBadChannel) + } + sub := &feedSub{feed: f, channel: chanval, err: make(chan error, 1)} + + f.mu.Lock() + defer f.mu.Unlock() + f.init() + if !f.typecheck(chantyp.Elem()) { + panic(feedTypeError{op: "Subscribe", got: chantyp, want: reflect.ChanOf(reflect.SendDir, f.etype)}) + } + // Add the select case to the inbox. + // The next Send will add it to f.sendCases. + cas := reflect.SelectCase{Dir: reflect.SelectSend, Chan: chanval} + f.inbox = append(f.inbox, cas) + return sub +} + +// note: callers must hold f.mu +func (f *Feed) typecheck(typ reflect.Type) bool { + if f.etype == nil { + f.etype = typ + return true + } + return f.etype == typ +} + +func (f *Feed) remove(sub *feedSub) { + // Delete from inbox first, which covers channels + // that have not been added to f.sendCases yet. + ch := sub.channel.Interface() + f.mu.Lock() + index := f.inbox.find(ch) + if index != -1 { + f.inbox = f.inbox.delete(index) + f.mu.Unlock() + return + } + f.mu.Unlock() + + select { + case f.removeSub <- ch: + // Send will remove the channel from f.sendCases. + case <-f.sendLock: + // No Send is in progress, delete the channel now that we have the send lock. + f.sendCases = f.sendCases.delete(f.sendCases.find(ch)) + f.sendLock <- struct{}{} + } +} + +// Send delivers to all subscribed channels simultaneously. +// It returns the number of subscribers that the value was sent to. +func (f *Feed) Send(value interface{}) (nsent int) { + f.mu.Lock() + f.init() + <-f.sendLock + // Add new subscriptions from the inbox, then clear it. + f.sendCases = append(f.sendCases, f.inbox...) + for i := range f.inbox { + f.inbox[i] = reflect.SelectCase{} + } + f.inbox = f.inbox[:0] + f.mu.Unlock() + + // Set the sent value on all channels. + rvalue := reflect.ValueOf(value) + if !f.typecheck(rvalue.Type()) { + f.sendLock <- struct{}{} + panic(feedTypeError{op: "Send", got: rvalue.Type(), want: f.etype}) + } + for i := 1; i < len(f.sendCases); i++ { + f.sendCases[i].Send = rvalue + } + + // Send until all channels except removeSub have been chosen. + cases := f.sendCases + for { + // Fast path: try sending without blocking before adding to the select set. + // This should usually succeed if subscribers are fast enough and have free + // buffer space. + for i := 1; i < len(cases); i++ { + if cases[i].Chan.TrySend(rvalue) { + cases = cases.deactivate(i) + nsent++ + } + } + if len(cases) == 1 { + break + } + // Select on all the receivers, waiting for them to unblock. + chosen, recv, _ := reflect.Select(cases) + if chosen == 0 /* <-f.removeSub */ { + index := f.sendCases.find(recv.Interface()) + f.sendCases = f.sendCases.delete(index) + if index >= 0 && index < len(cases) { + cases = f.sendCases[:len(cases)-1] + } + } else { + cases = cases.deactivate(chosen) + nsent++ + } + } + + // Forget about the sent value and hand off the send lock. + for i := 1; i < len(f.sendCases); i++ { + f.sendCases[i].Send = reflect.Value{} + } + f.sendLock <- struct{}{} + return nsent +} + +type feedSub struct { + feed *Feed + channel reflect.Value + errOnce sync.Once + err chan error +} + +func (sub *feedSub) Unsubscribe() { + sub.errOnce.Do(func() { + sub.feed.remove(sub) + close(sub.err) + }) +} + +func (sub *feedSub) Err() <-chan error { + return sub.err +} + +type caseList []reflect.SelectCase + +// find returns the index of a case containing the given channel. +func (cs caseList) find(channel interface{}) int { + for i, cas := range cs { + if cas.Chan.Interface() == channel { + return i + } + } + return -1 +} + +// delete removes the given case from cs. +func (cs caseList) delete(index int) caseList { + return append(cs[:index], cs[index+1:]...) +} + +// deactivate moves the case at index into the non-accessible portion of the cs slice. +func (cs caseList) deactivate(index int) caseList { + last := len(cs) - 1 + cs[index], cs[last] = cs[last], cs[index] + return cs[:last] +} + +// func (cs caseList) String() string { +// s := "[" +// for i, cas := range cs { +// if i != 0 { +// s += ", " +// } +// switch cas.Dir { +// case reflect.SelectSend: +// s += fmt.Sprintf("%v<-", cas.Chan.Interface()) +// case reflect.SelectRecv: +// s += fmt.Sprintf("<-%v", cas.Chan.Interface()) +// } +// } +// return s + "]" +// } diff --git a/event/feed_test.go b/event/feed_test.go new file mode 100644 index 0000000000..4f897c162c --- /dev/null +++ b/event/feed_test.go @@ -0,0 +1,226 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event + +import ( + "fmt" + "reflect" + "sync" + "testing" + "time" +) + +func TestFeedPanics(t *testing.T) { + { + var f Feed + f.Send(int(2)) + want := feedTypeError{op: "Send", got: reflect.TypeOf(uint64(0)), want: reflect.TypeOf(int(0))} + if err := checkPanic(want, func() { f.Send(uint64(2)) }); err != nil { + t.Error(err) + } + } + { + var f Feed + ch := make(chan int) + f.Subscribe(ch) + want := feedTypeError{op: "Send", got: reflect.TypeOf(uint64(0)), want: reflect.TypeOf(int(0))} + if err := checkPanic(want, func() { f.Send(uint64(2)) }); err != nil { + t.Error(err) + } + } + { + var f Feed + f.Send(int(2)) + want := feedTypeError{op: "Subscribe", got: reflect.TypeOf(make(chan uint64)), want: reflect.TypeOf(make(chan<- int))} + if err := checkPanic(want, func() { f.Subscribe(make(chan uint64)) }); err != nil { + t.Error(err) + } + } + { + var f Feed + if err := checkPanic(errBadChannel, func() { f.Subscribe(make(<-chan int)) }); err != nil { + t.Error(err) + } + } + { + var f Feed + if err := checkPanic(errBadChannel, func() { f.Subscribe(int(0)) }); err != nil { + t.Error(err) + } + } +} + +func checkPanic(want error, fn func()) (err error) { + defer func() { + panic := recover() + if panic == nil { + err = fmt.Errorf("didn't panic") + } else if !reflect.DeepEqual(panic, want) { + err = fmt.Errorf("panicked with wrong error: got %q, want %q", panic, want) + } + }() + fn() + return nil +} + +func TestFeed(t *testing.T) { + var feed Feed + var done, subscribed sync.WaitGroup + subscriber := func(i int) { + defer done.Done() + + subchan := make(chan int) + sub := feed.Subscribe(subchan) + timeout := time.NewTimer(2 * time.Second) + subscribed.Done() + + select { + case v := <-subchan: + if v != 1 { + t.Errorf("%d: received value %d, want 1", i, v) + } + case <-timeout.C: + t.Errorf("%d: receive timeout", i) + } + + sub.Unsubscribe() + select { + case _, ok := <-sub.Err(): + if ok { + t.Errorf("%d: error channel not closed after unsubscribe", i) + } + case <-timeout.C: + t.Errorf("%d: unsubscribe timeout", i) + } + } + + const n = 1000 + done.Add(n) + subscribed.Add(n) + for i := 0; i < n; i++ { + go subscriber(i) + } + subscribed.Wait() + if nsent := feed.Send(1); nsent != n { + t.Errorf("first send delivered %d times, want %d", nsent, n) + } + if nsent := feed.Send(2); nsent != 0 { + t.Errorf("second send delivered %d times, want 0", nsent) + } + done.Wait() +} + +func TestFeedSubscribeSameChannel(t *testing.T) { + var ( + feed Feed + done sync.WaitGroup + ch = make(chan int) + sub1 = feed.Subscribe(ch) + sub2 = feed.Subscribe(ch) + _ = feed.Subscribe(ch) + ) + expectSends := func(value, n int) { + if nsent := feed.Send(value); nsent != n { + t.Errorf("send delivered %d times, want %d", nsent, n) + } + done.Done() + } + expectRecv := func(wantValue, n int) { + for i := 0; i < n; i++ { + if v := <-ch; v != wantValue { + t.Errorf("received %d, want %d", v, wantValue) + } + } + } + + done.Add(1) + go expectSends(1, 3) + expectRecv(1, 3) + done.Wait() + + sub1.Unsubscribe() + + done.Add(1) + go expectSends(2, 2) + expectRecv(2, 2) + done.Wait() + + sub2.Unsubscribe() + + done.Add(1) + go expectSends(3, 1) + expectRecv(3, 1) + done.Wait() +} + +func TestFeedUnsubscribeFromInbox(t *testing.T) { + var ( + feed Feed + ch1 = make(chan int) + ch2 = make(chan int) + sub1 = feed.Subscribe(ch1) + sub2 = feed.Subscribe(ch1) + sub3 = feed.Subscribe(ch2) + ) + if len(feed.inbox) != 3 { + t.Errorf("inbox length != 3 after subscribe") + } + if len(feed.sendCases) != 1 { + t.Errorf("sendCases is non-empty after unsubscribe") + } + + sub1.Unsubscribe() + sub2.Unsubscribe() + sub3.Unsubscribe() + if len(feed.inbox) != 0 { + t.Errorf("inbox is non-empty after unsubscribe") + } + if len(feed.sendCases) != 1 { + t.Errorf("sendCases is non-empty after unsubscribe") + } +} + +func BenchmarkFeedSend1000(b *testing.B) { + var ( + done sync.WaitGroup + feed Feed + nsubs = 1000 + ) + subscriber := func(ch <-chan int) { + for i := 0; i < b.N; i++ { + <-ch + } + done.Done() + } + done.Add(nsubs) + for i := 0; i < nsubs; i++ { + ch := make(chan int, 200) + feed.Subscribe(ch) + go subscriber(ch) + } + + // The actual benchmark. + b.ResetTimer() + for i := 0; i < b.N; i++ { + if feed.Send(i) != nsubs { + panic("wrong number of sends") + } + } + + b.StopTimer() + done.Wait() +} diff --git a/event/subscription.go b/event/subscription.go new file mode 100644 index 0000000000..7f2619b2d7 --- /dev/null +++ b/event/subscription.go @@ -0,0 +1,275 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event + +import ( + "sync" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "golang.org/x/net/context" +) + +// Subscription represents a stream of events. The carrier of the events is typically a +// channel, but isn't part of the interface. +// +// Subscriptions can fail while established. Failures are reported through an error +// channel. It receives a value if there is an issue with the subscription (e.g. the +// network connection delivering the events has been closed). Only one value will ever be +// sent. +// +// The error channel is closed when the subscription ends successfully (i.e. when the +// source of events is closed). It is also closed when Unsubscribe is called. +// +// The Unsubscribe method cancels the sending of events. You must call Unsubscribe in all +// cases to ensure that resources related to the subscription are released. It can be +// called any number of times. +type Subscription interface { + Err() <-chan error // returns the error channel + Unsubscribe() // cancels sending of events, closing the error channel +} + +// NewSubscription runs fn as a subscription in a new goroutine. The channel given to fn +// is closed when Unsubscribe is called. If fn returns an error, it is sent on the +// subscription's error channel. +func NewSubscription(fn func(<-chan struct{}) error) Subscription { + s := &funcSub{unsub: make(chan struct{}), err: make(chan error, 1)} + go func() { + defer close(s.err) + err := fn(s.unsub) + s.mu.Lock() + defer s.mu.Unlock() + if !s.unsubscribed { + if err != nil { + s.err <- err + } + s.unsubscribed = true + } + }() + return s +} + +type funcSub struct { + unsub chan struct{} + err chan error + mu sync.Mutex + unsubscribed bool +} + +func (s *funcSub) Unsubscribe() { + s.mu.Lock() + if s.unsubscribed { + s.mu.Unlock() + return + } + s.unsubscribed = true + close(s.unsub) + s.mu.Unlock() + // Wait for producer shutdown. + <-s.err +} + +func (s *funcSub) Err() <-chan error { + return s.err +} + +// Resubscribe calls fn repeatedly to keep a subscription established. When the +// subscription is established, Resubscribe waits for it to fail and calls fn again. This +// process repeats until Unsubscribe is called or the active subscription ends +// successfully. +// +// Resubscribe applies backoff between calls to fn. The time between calls is adapted +// based on the error rate, but will never exceed backoffMax. +func Resubscribe(backoffMax time.Duration, fn ResubscribeFunc) Subscription { + s := &resubscribeSub{ + waitTime: backoffMax / 10, + backoffMax: backoffMax, + fn: fn, + err: make(chan error), + unsub: make(chan struct{}), + } + go s.loop() + return s +} + +// A ResubscribeFunc attempts to establish a subscription. +type ResubscribeFunc func(context.Context) (Subscription, error) + +type resubscribeSub struct { + fn ResubscribeFunc + err chan error + unsub chan struct{} + unsubOnce sync.Once + lastTry mclock.AbsTime + waitTime, backoffMax time.Duration +} + +func (s *resubscribeSub) Unsubscribe() { + s.unsubOnce.Do(func() { + s.unsub <- struct{}{} + <-s.err + }) +} + +func (s *resubscribeSub) Err() <-chan error { + return s.err +} + +func (s *resubscribeSub) loop() { + defer close(s.err) + var done bool + for !done { + sub := s.subscribe() + if sub == nil { + break + } + done = s.waitForError(sub) + sub.Unsubscribe() + } +} + +func (s *resubscribeSub) subscribe() Subscription { + subscribed := make(chan error) + var sub Subscription +retry: + for { + s.lastTry = mclock.Now() + ctx, cancel := context.WithCancel(context.Background()) + go func() { + rsub, err := s.fn(ctx) + sub = rsub + subscribed <- err + }() + select { + case err := <-subscribed: + cancel() + if err != nil { + // Subscribing failed, wait before launching the next try. + if s.backoffWait() { + return nil + } + continue retry + } + if sub == nil { + panic("event: ResubscribeFunc returned nil subscription and no error") + } + return sub + case <-s.unsub: + cancel() + return nil + } + } +} + +func (s *resubscribeSub) waitForError(sub Subscription) bool { + defer sub.Unsubscribe() + select { + case err := <-sub.Err(): + return err == nil + case <-s.unsub: + return true + } +} + +func (s *resubscribeSub) backoffWait() bool { + if time.Duration(mclock.Now()-s.lastTry) > s.backoffMax { + s.waitTime = s.backoffMax / 10 + } else { + s.waitTime *= 2 + if s.waitTime > s.backoffMax { + s.waitTime = s.backoffMax + } + } + + t := time.NewTimer(s.waitTime) + defer t.Stop() + select { + case <-t.C: + return false + case <-s.unsub: + return true + } +} + +// SubscriptionScope provides a facility to unsubscribe multiple subscriptions at once. +// +// For code that handle more than one subscription, a scope can be used to conveniently +// unsubscribe all of them with a single call. The example demonstrates a typical use in a +// larger program. +// +// The zero value is ready to use. +type SubscriptionScope struct { + mu sync.Mutex + subs map[*scopeSub]struct{} + closed bool +} + +type scopeSub struct { + sc *SubscriptionScope + s Subscription +} + +// Track starts tracking a subscription. If the scope is closed, Track returns nil. The +// returned subscription is a wrapper. Unsubscribing the wrapper removes it from the +// scope. +func (sc *SubscriptionScope) Track(s Subscription) Subscription { + sc.mu.Lock() + defer sc.mu.Unlock() + if sc.closed { + return nil + } + if sc.subs == nil { + sc.subs = make(map[*scopeSub]struct{}) + } + ss := &scopeSub{sc, s} + sc.subs[ss] = struct{}{} + return ss +} + +// Close calls Unsubscribe on all tracked subscriptions and prevents further additions to +// the tracked set. Calls to Track after Close return nil. +func (sc *SubscriptionScope) Close() { + sc.mu.Lock() + defer sc.mu.Unlock() + if sc.closed { + return + } + sc.closed = true + for s := range sc.subs { + s.s.Unsubscribe() + } + sc.subs = nil +} + +// Count returns the number of tracked subscriptions. +// It is meant to be used for debugging. +func (sc *SubscriptionScope) Count() int { + sc.mu.Lock() + defer sc.mu.Unlock() + return len(sc.subs) +} + +func (s *scopeSub) Unsubscribe() { + s.s.Unsubscribe() + s.sc.mu.Lock() + defer s.sc.mu.Unlock() + delete(s.sc.subs, s) +} + +func (s *scopeSub) Err() <-chan error { + return s.s.Err() +} diff --git a/event/subscription_test.go b/event/subscription_test.go new file mode 100644 index 0000000000..a4fe30298c --- /dev/null +++ b/event/subscription_test.go @@ -0,0 +1,121 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event + +import ( + "errors" + "testing" + "time" + + "golang.org/x/net/context" +) + +var errInts = errors.New("error in subscribeInts") + +func subscribeInts(max, fail int, c chan<- int) Subscription { + return NewSubscription(func(quit <-chan struct{}) error { + for i := 0; i < max; i++ { + if i >= fail { + return errInts + } + select { + case c <- i: + case <-quit: + return nil + } + } + return nil + }) +} + +func TestNewSubscriptionError(t *testing.T) { + t.Parallel() + + channel := make(chan int) + sub := subscribeInts(10, 2, channel) +loop: + for want := 0; want < 10; want++ { + select { + case got := <-channel: + if got != want { + t.Fatalf("wrong int %d, want %d", got, want) + } + case err := <-sub.Err(): + if err != errInts { + t.Fatalf("wrong error: got %q, want %q", err, errInts) + } + if want != 2 { + t.Fatalf("got errInts at int %d, should be received at 2", want) + } + break loop + } + } + sub.Unsubscribe() + + err, ok := <-sub.Err() + if err != nil { + t.Fatal("got non-nil error after Unsubscribe") + } + if ok { + t.Fatal("channel still open after Unsubscribe") + } +} + +func TestResubscribe(t *testing.T) { + t.Parallel() + + var i int + nfails := 6 + sub := Resubscribe(100*time.Millisecond, func(ctx context.Context) (Subscription, error) { + // fmt.Printf("call #%d @ %v\n", i, time.Now()) + i++ + if i == 2 { + // Delay the second failure a bit to reset the resubscribe interval. + time.Sleep(200 * time.Millisecond) + } + if i < nfails { + return nil, errors.New("oops") + } + sub := NewSubscription(func(unsubscribed <-chan struct{}) error { return nil }) + return sub, nil + }) + + <-sub.Err() + if i != nfails { + t.Fatalf("resubscribe function called %d times, want %d times", i, nfails) + } +} + +func TestResubscribeAbort(t *testing.T) { + t.Parallel() + + done := make(chan error) + sub := Resubscribe(0, func(ctx context.Context) (Subscription, error) { + select { + case <-ctx.Done(): + done <- nil + case <-time.After(2 * time.Second): + done <- errors.New("context given to resubscribe function not canceled within 2s") + } + return nil, nil + }) + + sub.Unsubscribe() + if err := <-done; err != nil { + t.Fatal(err) + } +} From a2b4abd89adf0404b43fcd19766bac37009032a5 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 9 Dec 2016 19:51:21 +0100 Subject: [PATCH 04/21] rpc: send nil on subscription Err channel when Client is closed This change makes client subscriptions compatible with the new Subscription semantics introduced in the previous commit. --- rpc/client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rpc/client.go b/rpc/client.go index 34a3b78317..269eb78c86 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -682,7 +682,7 @@ func newClientSubscription(c *Client, channel reflect.Value) *ClientSubscription // resubscription when the client connection is closed unexpectedly. // // The error channel receives a value when the subscription has ended due -// to an error. The received error is ErrClientQuit if Close has been called +// to an error. The received error is nil if Close has been called // on the underlying client and no other error has occurred. // // The error channel is closed when Unsubscribe is called on the subscription. @@ -707,6 +707,9 @@ func (sub *ClientSubscription) quitWithError(err error, unsubscribeServer bool) sub.requestUnsubscribe() } if err != nil { + if err == ErrClientQuit { + err = nil // Adhere to subscription semantics. + } sub.err <- err } }) From f5348e17f8d487a7bd34ccef6dcb6dc376407f40 Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Mon, 23 Jan 2017 01:54:01 +0100 Subject: [PATCH 05/21] les: add unknown peers to server pool instead of rejecting them --- les/handler.go | 3 -- les/serverpool.go | 80 ++++++++++++++++++++++++++--------------------- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/les/handler.go b/les/handler.go index 603ce9ad4f..42a45845d0 100644 --- a/les/handler.go +++ b/les/handler.go @@ -160,9 +160,6 @@ func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, network if manager.serverPool != nil { addr := p.RemoteAddr().(*net.TCPAddr) entry = manager.serverPool.connect(peer, addr.IP, uint16(addr.Port)) - if entry == nil { - return fmt.Errorf("unwanted connection") - } } peer.poolEntry = entry select { diff --git a/les/serverpool.go b/les/serverpool.go index e3b7cf6200..68e962c978 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -160,10 +160,10 @@ func (pool *serverPool) connect(p *peer, ip net.IP, port uint16) *poolEntry { defer pool.lock.Unlock() entry := pool.entries[p.ID()] if entry == nil { - return nil + entry = pool.findOrNewNode(p.ID(), ip, port) } glog.V(logger.Debug).Infof("connecting to %v, state: %v", p.id, entry.state) - if entry.state != psDialed { + if entry.state == psConnected || entry.state == psRegistered { return nil } pool.connWg.Add(1) @@ -250,11 +250,17 @@ type poolStatAdjust struct { // adjustBlockDelay adjusts the block announce delay statistics of a node func (pool *serverPool) adjustBlockDelay(entry *poolEntry, time time.Duration) { + if entry == nil { + return + } pool.adjustStats <- poolStatAdjust{pseBlockDelay, entry, time} } // adjustResponseTime adjusts the request response time statistics of a node func (pool *serverPool) adjustResponseTime(entry *poolEntry, time time.Duration, timeout bool) { + if entry == nil { + return + } if timeout { pool.adjustStats <- poolStatAdjust{pseResponseTimeout, entry, time} } else { @@ -375,39 +381,7 @@ func (pool *serverPool) eventLoop() { case node := <-pool.discNodes: pool.lock.Lock() - now := mclock.Now() - id := discover.NodeID(node.ID) - entry := pool.entries[id] - if entry == nil { - glog.V(logger.Debug).Infof("discovered %v", node.String()) - entry = &poolEntry{ - id: id, - addr: make(map[string]*poolEntryAddress), - addrSelect: *newWeightedRandomSelect(), - shortRetry: shortRetryCnt, - } - pool.entries[id] = entry - // initialize previously unknown peers with good statistics to give a chance to prove themselves - entry.connectStats.add(1, initStatsWeight) - entry.delayStats.add(0, initStatsWeight) - entry.responseStats.add(0, initStatsWeight) - entry.timeoutStats.add(0, initStatsWeight) - } - entry.lastDiscovered = now - addr := &poolEntryAddress{ - ip: node.IP, - port: node.TCP, - } - if a, ok := entry.addr[addr.strKey()]; ok { - addr = a - } else { - entry.addr[addr.strKey()] = addr - } - addr.lastSeen = now - entry.addrSelect.update(addr) - if !entry.known { - pool.newQueue.setLatest(entry) - } + entry := pool.findOrNewNode(discover.NodeID(node.ID), node.IP, node.TCP) pool.updateCheckDial(entry) pool.lock.Unlock() @@ -434,6 +408,42 @@ func (pool *serverPool) eventLoop() { } } +func (pool *serverPool) findOrNewNode(id discover.NodeID, ip net.IP, port uint16) *poolEntry { + now := mclock.Now() + entry := pool.entries[id] + if entry == nil { + glog.V(logger.Debug).Infof("discovered %v", id.String()) + entry = &poolEntry{ + id: id, + addr: make(map[string]*poolEntryAddress), + addrSelect: *newWeightedRandomSelect(), + shortRetry: shortRetryCnt, + } + pool.entries[id] = entry + // initialize previously unknown peers with good statistics to give a chance to prove themselves + entry.connectStats.add(1, initStatsWeight) + entry.delayStats.add(0, initStatsWeight) + entry.responseStats.add(0, initStatsWeight) + entry.timeoutStats.add(0, initStatsWeight) + } + entry.lastDiscovered = now + addr := &poolEntryAddress{ + ip: ip, + port: port, + } + if a, ok := entry.addr[addr.strKey()]; ok { + addr = a + } else { + entry.addr[addr.strKey()] = addr + } + addr.lastSeen = now + entry.addrSelect.update(addr) + if !entry.known { + pool.newQueue.setLatest(entry) + } + return entry +} + // loadNodes loads known nodes and their statistics from the database func (pool *serverPool) loadNodes() { enc, err := pool.db.Get(pool.dbKey) From 12379c697aa72f4ce08733638e6f7e5dcdf3f94f Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Mon, 23 Jan 2017 02:35:46 +0100 Subject: [PATCH 06/21] les: remove delayed les server starting --- eth/backend.go | 1 - eth/handler.go | 2 +- eth/sync.go | 9 +-------- les/server.go | 30 +++--------------------------- 4 files changed, 5 insertions(+), 37 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index e0233db363..af120cbadd 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -105,7 +105,6 @@ type Config struct { type LesServer interface { Start(srvr *p2p.Server) - Synced() Stop() Protocols() []p2p.Protocol } diff --git a/eth/handler.go b/eth/handler.go index 63ba0821fe..e03c891493 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -173,7 +173,7 @@ func NewProtocolManager(config *params.ChainConfig, fastSync bool, networkId int return blockchain.CurrentBlock().NumberU64() } inserter := func(blocks types.Blocks) (int, error) { - manager.setSynced() // Mark initial sync done on any fetcher import + atomic.StoreUint32(&manager.synced, 1) // Mark initial sync done on any fetcher import return manager.insertChain(blocks) } manager.fetcher = fetcher.New(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) diff --git a/eth/sync.go b/eth/sync.go index 234534b4fb..373cc2054b 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -181,7 +181,7 @@ func (pm *ProtocolManager) synchronise(peer *peer) { if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil { return } - pm.setSynced() // Mark initial sync done + atomic.StoreUint32(&pm.synced, 1) // Mark initial sync done // If fast sync was enabled, and we synced up, disable it if atomic.LoadUint32(&pm.fastSync) == 1 { @@ -192,10 +192,3 @@ func (pm *ProtocolManager) synchronise(peer *peer) { } } } - -// setSynced sets the synced flag and notifies the light server if present -func (pm *ProtocolManager) setSynced() { - if atomic.SwapUint32(&pm.synced, 1) == 0 && pm.lesServer != nil { - pm.lesServer.Synced() - } -} diff --git a/les/server.go b/les/server.go index e55616a444..c4c6fcab52 100644 --- a/les/server.go +++ b/les/server.go @@ -42,9 +42,7 @@ type LesServer struct { fcManager *flowcontrol.ClientManager // nil if our node is client only fcCostStats *requestCostStats defParams *flowcontrol.ServerParams - srvr *p2p.Server - synced, stopped bool - lock sync.Mutex + stopped bool } func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { @@ -70,35 +68,13 @@ func (s *LesServer) Protocols() []p2p.Protocol { return s.protocolManager.SubProtocols } -// Start only starts the actual service if the ETH protocol has already been synced, -// otherwise it will be started by Synced() +// Start starts the LES server func (s *LesServer) Start(srvr *p2p.Server) { - s.lock.Lock() - defer s.lock.Unlock() - - s.srvr = srvr - if s.synced { - s.protocolManager.Start(s.srvr) - } -} - -// Synced notifies the server that the ETH protocol has been synced and LES service can be started -func (s *LesServer) Synced() { - s.lock.Lock() - defer s.lock.Unlock() - - s.synced = true - if s.srvr != nil && !s.stopped { - s.protocolManager.Start(s.srvr) - } + s.protocolManager.Start(srvr) } // Stop stops the LES service func (s *LesServer) Stop() { - s.lock.Lock() - defer s.lock.Unlock() - - s.stopped = true s.fcCostStats.store() s.fcManager.Stop() go func() { From a390ca5f30e871715f28937a02e87fff050b0c75 Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Thu, 26 Jan 2017 02:52:59 +0100 Subject: [PATCH 07/21] les, cmd/util: disable topic discovery with --nodiscover --- cmd/utils/flags.go | 8 ++++++-- les/serverpool.go | 12 +++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 4b76b83347..9ba33df809 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -654,6 +654,10 @@ func MakeNode(ctx *cli.Context, name, gitCommit string) *node.Node { vsn += "-" + gitCommit[:8] } + // if we're running a light client or server, force enable the v5 peer discovery unless it is explicitly disabled with --nodiscover + // note that explicitly specifying --v5disc overrides --nodiscover, in which case the later only disables v4 discovery + forceV5Discovery := (ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalInt(LightServFlag.Name) > 0) && !ctx.GlobalBool(NoDiscoverFlag.Name) + config := &node.Config{ DataDir: MakeDataDir(ctx), KeyStoreDir: ctx.GlobalString(KeyStoreDirFlag.Name), @@ -662,8 +666,8 @@ func MakeNode(ctx *cli.Context, name, gitCommit string) *node.Node { Name: name, Version: vsn, UserIdent: makeNodeUserIdent(ctx), - NoDiscovery: ctx.GlobalBool(NoDiscoverFlag.Name) || ctx.GlobalBool(LightModeFlag.Name), - DiscoveryV5: ctx.GlobalBool(DiscoveryV5Flag.Name) || ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalInt(LightServFlag.Name) > 0, + NoDiscovery: ctx.GlobalBool(NoDiscoverFlag.Name) || ctx.GlobalBool(LightModeFlag.Name), // always disable v4 discovery in light client mode + DiscoveryV5: ctx.GlobalBool(DiscoveryV5Flag.Name) || forceV5Discovery, DiscoveryV5Addr: MakeDiscoveryV5Address(ctx), BootstrapNodes: MakeBootstrapNodes(ctx), BootstrapNodesV5: MakeBootstrapNodesV5(ctx), diff --git a/les/serverpool.go b/les/serverpool.go index 68e962c978..9735a718e5 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -348,7 +348,9 @@ func (pool *serverPool) selectPeerWait(reqID uint64, canSend func(*peer) (bool, func (pool *serverPool) eventLoop() { lookupCnt := 0 var convTime mclock.AbsTime - pool.discSetPeriod <- time.Millisecond * 100 + if pool.discSetPeriod != nil { + pool.discSetPeriod <- time.Millisecond * 100 + } for { select { case entry := <-pool.timeout: @@ -393,12 +395,16 @@ func (pool *serverPool) eventLoop() { lookupCnt++ if pool.fastDiscover && (lookupCnt == 50 || time.Duration(mclock.Now()-convTime) > time.Minute) { pool.fastDiscover = false - pool.discSetPeriod <- time.Minute + if pool.discSetPeriod != nil { + pool.discSetPeriod <- time.Minute + } } } case <-pool.quit: - close(pool.discSetPeriod) + if pool.discSetPeriod != nil { + close(pool.discSetPeriod) + } pool.connWg.Wait() pool.saveNodes() pool.wg.Done() From f58fb32283fe04cd1d416040c6692b4a7352d556 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 1 Feb 2017 02:14:04 +0100 Subject: [PATCH 08/21] params: v1.5.8-stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index b60e105d56..cc95f5de30 100644 --- a/params/version.go +++ b/params/version.go @@ -19,10 +19,10 @@ package params import "fmt" const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 5 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 5 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 4f5f90222f06f87729a1f05f09472857fa9a32a7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 1 Feb 2017 02:14:54 +0100 Subject: [PATCH 09/21] params, VERSION: v1.5.9-unstable --- VERSION | 2 +- params/version.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/VERSION b/VERSION index 1cc9c180e2..2b26b8d213 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.8 +1.5.9 diff --git a/params/version.go b/params/version.go index cc95f5de30..72ec2d239a 100644 --- a/params/version.go +++ b/params/version.go @@ -19,10 +19,10 @@ package params import "fmt" const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 5 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 5 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 296450451b090393b2dd11d057c5b72cb4d92356 Mon Sep 17 00:00:00 2001 From: Brian Schroeder Date: Wed, 1 Feb 2017 04:55:46 -0500 Subject: [PATCH 10/21] state: take write lock in GetNonce (#3625) We must take a write lock here because `GetNonce` calls `StateDB.GetStateObject`, which mutates the DB's live set. --- core/state/managed_state.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/state/managed_state.go b/core/state/managed_state.go index ad73dc0dc6..0d8f9dd284 100644 --- a/core/state/managed_state.go +++ b/core/state/managed_state.go @@ -82,10 +82,12 @@ func (ms *ManagedState) NewNonce(addr common.Address) uint64 { return uint64(len(account.nonces)-1) + account.nstart } -// GetNonce returns the canonical nonce for the managed or unmanaged account +// GetNonce returns the canonical nonce for the managed or unmanaged account. +// +// Because GetNonce mutates the DB, we must take a write lock. func (ms *ManagedState) GetNonce(addr common.Address) uint64 { - ms.mu.RLock() - defer ms.mu.RUnlock() + ms.mu.Lock() + defer ms.mu.Unlock() if ms.hasAccount(addr) { account := ms.getAccount(addr) From 8b57c494908637a5c0e74f8f7a13b3218e026757 Mon Sep 17 00:00:00 2001 From: Jeffrey Wilcke Date: Thu, 2 Feb 2017 15:25:42 +0100 Subject: [PATCH 11/21] params: core, core/vm, miner: 64bit gas instructions (#3514) Reworked the EVM gas instructions to use 64bit integers rather than arbitrary size big ints. All gas operations, be it additions, multiplications or divisions, are checked and guarded against 64 bit integer overflows. In additon, most of the protocol paramaters in the params package have been converted to uint64 and are now constants rather than variables. * common/math: added overflow check ops * core: vmenv, env renamed to evm * eth, internal/ethapi, les: unmetered eth_call and cancel methods * core/vm: implemented big.Int pool for evm instructions * core/vm: unexported intPool methods & verification methods * core/vm: added memoryGasCost overflow check and test --- cmd/evm/main.go | 4 +- cmd/geth/main.go | 2 +- common/math/integer.go | 25 ++ common/math/integer_test.go | 50 +++ core/bench_test.go | 7 +- core/block_validator.go | 2 +- core/blockchain_test.go | 16 +- core/chain_makers_test.go | 6 +- core/state_transition.go | 86 +++-- core/vm/common.go | 78 +--- core/vm/contract.go | 33 +- core/vm/contracts.go | 39 +- core/vm/{environment.go => evm.go} | 83 ++-- core/vm/gas.go | 151 ++------ core/vm/gas_table.go | 420 +++++++++++++++------ core/vm/gas_table_test.go | 24 ++ core/vm/instructions.go | 344 ++++++++++------- core/vm/int_pool_verifier.go | 15 + core/vm/int_pool_verifier_empty.go | 7 + core/vm/{vm.go => interpreter.go} | 42 ++- core/vm/{virtual_machine.go => intpool.go} | 35 +- core/vm/jump_table.go | 7 +- core/vm/logger_test.go | 4 +- core/vm/memory.go | 5 +- core/vm/runtime/env.go | 2 +- core/vm/runtime/runtime.go | 14 +- core/vm/runtime/runtime_test.go | 4 +- core/vm/stack_table.go | 4 +- eth/api_backend.go | 4 +- eth/bind.go | 4 +- eth/downloader/downloader.go | 12 +- eth/downloader/downloader_test.go | 2 +- eth/fetcher/fetcher_test.go | 2 +- eth/handler_test.go | 14 +- internal/ethapi/api.go | 72 ++-- internal/ethapi/backend.go | 2 +- internal/ethapi/tracer_test.go | 4 +- les/api_backend.go | 4 +- les/helper_test.go | 8 +- light/odr_test.go | 8 +- light/txpool_test.go | 2 +- miner/miner.go | 2 +- params/gas_table.go | 68 ++-- params/protocol_params.go | 100 ++--- tests/block_test_util.go | 2 +- tests/state_test_util.go | 2 +- tests/vm_test_util.go | 6 +- 47 files changed, 1043 insertions(+), 784 deletions(-) create mode 100644 common/math/integer.go create mode 100644 common/math/integer_test.go rename core/vm/{environment.go => evm.go} (85%) create mode 100644 core/vm/gas_table_test.go create mode 100644 core/vm/int_pool_verifier.go create mode 100644 core/vm/int_pool_verifier_empty.go rename core/vm/{vm.go => interpreter.go} (78%) rename core/vm/{virtual_machine.go => intpool.go} (53%) diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 9f67e66286..8af0cebbb5 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -156,7 +156,7 @@ func run(ctx *cli.Context) error { ret, _, err = runtime.Create(input, &runtime.Config{ Origin: sender.Address(), State: statedb, - GasLimit: common.Big(ctx.GlobalString(GasFlag.Name)), + GasLimit: common.Big(ctx.GlobalString(GasFlag.Name)).Uint64(), GasPrice: common.Big(ctx.GlobalString(PriceFlag.Name)), Value: common.Big(ctx.GlobalString(ValueFlag.Name)), EVMConfig: vm.Config{ @@ -172,7 +172,7 @@ func run(ctx *cli.Context) error { ret, err = runtime.Call(receiver.Address(), common.Hex2Bytes(ctx.GlobalString(InputFlag.Name)), &runtime.Config{ Origin: sender.Address(), State: statedb, - GasLimit: common.Big(ctx.GlobalString(GasFlag.Name)), + GasLimit: common.Big(ctx.GlobalString(GasFlag.Name)).Uint64(), GasPrice: common.Big(ctx.GlobalString(PriceFlag.Name)), Value: common.Big(ctx.GlobalString(ValueFlag.Name)), EVMConfig: vm.Config{ diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ff9d34b3c0..d7e4cc7b53 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -202,7 +202,7 @@ func makeFullNode(ctx *cli.Context) *node.Node { if err != nil { glog.V(logger.Warn).Infoln("error setting canonical miner information:", err) } - if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() { + if uint64(len(extra)) > params.MaximumExtraDataSize { glog.V(logger.Warn).Infoln("error setting canonical miner information: extra exceeds", params.MaximumExtraDataSize) glog.V(logger.Debug).Infof("extra: %x\n", extra) extra = nil diff --git a/common/math/integer.go b/common/math/integer.go new file mode 100644 index 0000000000..8162b19854 --- /dev/null +++ b/common/math/integer.go @@ -0,0 +1,25 @@ +package math + +import gmath "math" + +/* + * NOTE: The following methods need to be optimised using either bit checking or asm + */ + +// SafeSub returns subtraction result and whether overflow occurred. +func SafeSub(x, y uint64) (uint64, bool) { + return x - y, x < y +} + +// SafeAdd returns the result and whether overflow occurred. +func SafeAdd(x, y uint64) (uint64, bool) { + return x + y, y > gmath.MaxUint64-x +} + +// SafeMul returns multiplication result and whether overflow occurred. +func SafeMul(x, y uint64) (uint64, bool) { + if x == 0 { + return 0, false + } + return x * y, x != 0 && y != 0 && y > gmath.MaxUint64/x +} diff --git a/common/math/integer_test.go b/common/math/integer_test.go new file mode 100644 index 0000000000..198114e5e1 --- /dev/null +++ b/common/math/integer_test.go @@ -0,0 +1,50 @@ +package math + +import ( + gmath "math" + "testing" +) + +type operation byte + +const ( + sub operation = iota + add + mul +) + +func TestOverflow(t *testing.T) { + for i, test := range []struct { + x uint64 + y uint64 + overflow bool + op operation + }{ + // add operations + {gmath.MaxUint64, 1, true, add}, + {gmath.MaxUint64 - 1, 1, false, add}, + + // sub operations + {0, 1, true, sub}, + {0, 0, false, sub}, + + // mul operations + {10, 10, false, mul}, + {gmath.MaxUint64, 2, true, mul}, + {gmath.MaxUint64, 1, false, mul}, + } { + var overflows bool + switch test.op { + case sub: + _, overflows = SafeSub(test.x, test.y) + case add: + _, overflows = SafeAdd(test.x, test.y) + case mul: + _, overflows = SafeMul(test.x, test.y) + } + + if test.overflow != overflows { + t.Errorf("%d failed. Expected test to be %v, got %v", i, test.overflow, overflows) + } + } +} diff --git a/core/bench_test.go b/core/bench_test.go index 353d217fd4..59b5ad7589 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -92,6 +92,7 @@ func genValueTx(nbytes int) func(int, *BlockGen) { var ( ringKeys = make([]*ecdsa.PrivateKey, 1000) ringAddrs = make([]common.Address, len(ringKeys)) + bigTxGas = new(big.Int).SetUint64(params.TxGas) ) func init() { @@ -111,8 +112,8 @@ func genTxRing(naccounts int) func(int, *BlockGen) { return func(i int, gen *BlockGen) { gas := CalcGasLimit(gen.PrevBlock(i - 1)) for { - gas.Sub(gas, params.TxGas) - if gas.Cmp(params.TxGas) < 0 { + gas.Sub(gas, bigTxGas) + if gas.Cmp(bigTxGas) < 0 { break } to := (from + 1) % naccounts @@ -120,7 +121,7 @@ func genTxRing(naccounts int) func(int, *BlockGen) { gen.TxNonce(ringAddrs[from]), ringAddrs[to], benchRootFunds, - params.TxGas, + bigTxGas, nil, nil, ) diff --git a/core/block_validator.go b/core/block_validator.go index 65f975345c..ee524b61f8 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -204,7 +204,7 @@ func (v *BlockValidator) ValidateHeader(header, parent *types.Header, checkPow b // // See YP section 4.3.4. "Block Header Validity" func ValidateHeader(config *params.ChainConfig, pow pow.PoW, header *types.Header, parent *types.Header, checkPow, uncle bool) error { - if big.NewInt(int64(len(header.Extra))).Cmp(params.MaximumExtraDataSize) == 1 { + if uint64(len(header.Extra)) > params.MaximumExtraDataSize { return fmt.Errorf("Header extra data too long (%d)", len(header.Extra)) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 8f1383acdb..cdf9b5cc64 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -719,7 +719,7 @@ func TestFastVsFullChains(t *testing.T) { // If the block number is multiple of 3, send a few bonus transactions to the miner if i%3 == 2 { for j := 0; j < i%4+1; j++ { - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, nil, nil), signer, key) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), bigTxGas, nil, nil), signer, key) if err != nil { panic(err) } @@ -883,8 +883,8 @@ func TestChainTxReorgs(t *testing.T) { // Create two transactions shared between the chains: // - postponed: transaction included at a later block in the forked chain // - swapped: transaction included at the same block number in the forked chain - postponed, _ := types.SignTx(types.NewTransaction(0, addr1, big.NewInt(1000), params.TxGas, nil, nil), signer, key1) - swapped, _ := types.SignTx(types.NewTransaction(1, addr1, big.NewInt(1000), params.TxGas, nil, nil), signer, key1) + postponed, _ := types.SignTx(types.NewTransaction(0, addr1, big.NewInt(1000), bigTxGas, nil, nil), signer, key1) + swapped, _ := types.SignTx(types.NewTransaction(1, addr1, big.NewInt(1000), bigTxGas, nil, nil), signer, key1) // Create two transactions that will be dropped by the forked chain: // - pastDrop: transaction dropped retroactively from a past block @@ -900,13 +900,13 @@ func TestChainTxReorgs(t *testing.T) { chain, _ := GenerateChain(params.TestChainConfig, genesis, db, 3, func(i int, gen *BlockGen) { switch i { case 0: - pastDrop, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), params.TxGas, nil, nil), signer, key2) + pastDrop, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), bigTxGas, nil, nil), signer, key2) gen.AddTx(pastDrop) // This transaction will be dropped in the fork from below the split point gen.AddTx(postponed) // This transaction will be postponed till block #3 in the fork case 2: - freshDrop, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), params.TxGas, nil, nil), signer, key2) + freshDrop, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), bigTxGas, nil, nil), signer, key2) gen.AddTx(freshDrop) // This transaction will be dropped in the fork from exactly at the split point gen.AddTx(swapped) // This transaction will be swapped out at the exact height @@ -925,18 +925,18 @@ func TestChainTxReorgs(t *testing.T) { chain, _ = GenerateChain(params.TestChainConfig, genesis, db, 5, func(i int, gen *BlockGen) { switch i { case 0: - pastAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, nil, nil), signer, key3) + pastAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), bigTxGas, nil, nil), signer, key3) gen.AddTx(pastAdd) // This transaction needs to be injected during reorg case 2: gen.AddTx(postponed) // This transaction was postponed from block #1 in the original chain gen.AddTx(swapped) // This transaction was swapped from the exact current spot in the original chain - freshAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, nil, nil), signer, key3) + freshAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), bigTxGas, nil, nil), signer, key3) gen.AddTx(freshAdd) // This transaction will be added exactly at reorg time case 3: - futureAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, nil, nil), signer, key3) + futureAdd, _ = types.SignTx(types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), bigTxGas, nil, nil), signer, key3) gen.AddTx(futureAdd) // This transaction will be added after a full reorg } }) diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 2796817c01..5c563de469 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -56,13 +56,13 @@ func ExampleGenerateChain() { switch i { case 0: // In block 1, addr1 sends addr2 some ether. - tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(addr1), addr2, big.NewInt(10000), params.TxGas, nil, nil), signer, key1) + tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(addr1), addr2, big.NewInt(10000), bigTxGas, nil, nil), signer, key1) gen.AddTx(tx) case 1: // In block 2, addr1 sends some more ether to addr2. // addr2 passes it on to addr3. - tx1, _ := types.SignTx(types.NewTransaction(gen.TxNonce(addr1), addr2, big.NewInt(1000), params.TxGas, nil, nil), signer, key1) - tx2, _ := types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr3, big.NewInt(1000), params.TxGas, nil, nil), signer, key2) + tx1, _ := types.SignTx(types.NewTransaction(gen.TxNonce(addr1), addr2, big.NewInt(1000), bigTxGas, nil, nil), signer, key1) + tx2, _ := types.SignTx(types.NewTransaction(gen.TxNonce(addr2), addr3, big.NewInt(1000), bigTxGas, nil, nil), signer, key2) gen.AddTx(tx1) gen.AddTx(tx2) case 2: diff --git a/core/state_transition.go b/core/state_transition.go index 3cb7e500c5..98dc8d9958 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -49,15 +49,16 @@ The state transitioning model does all all the necessary work to work out a vali 6) Derive new state root */ type StateTransition struct { - gp *GasPool - msg Message - gas, gasPrice *big.Int - initialGas *big.Int - value *big.Int - data []byte - state vm.StateDB + gp *GasPool + msg Message + gas uint64 + gasPrice *big.Int + initialGas *big.Int + value *big.Int + data []byte + state vm.StateDB - env *vm.EVM + evm *vm.EVM } // Message represents a message sent to a contract. @@ -81,12 +82,14 @@ func MessageCreatesContract(msg Message) bool { // IntrinsicGas computes the 'intrinsic gas' for a message // with the given data. +// +// TODO convert to uint64 func IntrinsicGas(data []byte, contractCreation, homestead bool) *big.Int { igas := new(big.Int) if contractCreation && homestead { - igas.Set(params.TxGasContractCreation) + igas.SetUint64(params.TxGasContractCreation) } else { - igas.Set(params.TxGas) + igas.SetUint64(params.TxGas) } if len(data) > 0 { var nz int64 @@ -96,27 +99,26 @@ func IntrinsicGas(data []byte, contractCreation, homestead bool) *big.Int { } } m := big.NewInt(nz) - m.Mul(m, params.TxDataNonZeroGas) + m.Mul(m, new(big.Int).SetUint64(params.TxDataNonZeroGas)) igas.Add(igas, m) m.SetInt64(int64(len(data)) - nz) - m.Mul(m, params.TxDataZeroGas) + m.Mul(m, new(big.Int).SetUint64(params.TxDataZeroGas)) igas.Add(igas, m) } return igas } // NewStateTransition initialises and returns a new state transition object. -func NewStateTransition(env *vm.EVM, msg Message, gp *GasPool) *StateTransition { +func NewStateTransition(evm *vm.EVM, msg Message, gp *GasPool) *StateTransition { return &StateTransition{ gp: gp, - env: env, + evm: evm, msg: msg, - gas: new(big.Int), gasPrice: msg.GasPrice(), initialGas: new(big.Int), value: msg.Value(), data: msg.Data(), - state: env.StateDB, + state: evm.StateDB, } } @@ -127,8 +129,8 @@ func NewStateTransition(env *vm.EVM, msg Message, gp *GasPool) *StateTransition // the gas used (which includes gas refunds) and an error if it failed. An error always // indicates a core error meaning that the message would always fail for that particular // state and would never be accepted within a block. -func ApplyMessage(env *vm.EVM, msg Message, gp *GasPool) ([]byte, *big.Int, error) { - st := NewStateTransition(env, msg, gp) +func ApplyMessage(evm *vm.EVM, msg Message, gp *GasPool) ([]byte, *big.Int, error) { + st := NewStateTransition(evm, msg, gp) ret, _, gasUsed, err := st.TransitionDb() return ret, gasUsed, err @@ -157,21 +159,21 @@ func (self *StateTransition) to() vm.Account { return self.state.GetAccount(*to) } -func (self *StateTransition) useGas(amount *big.Int) error { - if self.gas.Cmp(amount) < 0 { +func (self *StateTransition) useGas(amount uint64) error { + if self.gas < amount { return vm.ErrOutOfGas } - self.gas.Sub(self.gas, amount) + self.gas -= amount return nil } -func (self *StateTransition) addGas(amount *big.Int) { - self.gas.Add(self.gas, amount) -} - func (self *StateTransition) buyGas() error { mgas := self.msg.Gas() + if mgas.BitLen() > 64 { + return vm.ErrOutOfGas + } + mgval := new(big.Int).Mul(mgas, self.gasPrice) sender := self.from() @@ -181,7 +183,8 @@ func (self *StateTransition) buyGas() error { if err := self.gp.SubGas(mgas); err != nil { return err } - self.addGas(mgas) + self.gas += mgas.Uint64() + self.initialGas.Set(mgas) sender.SubBalance(mgval) return nil @@ -209,7 +212,9 @@ func (self *StateTransition) preCheck() (err error) { return nil } -// TransitionDb will move the state by applying the message against the given environment. +// TransitionDb will transition the state by applying the current message and returning the result +// including the required gas for the operation as well as the used gas. It returns an error if it +// failed. An error indicates a consensus issue. func (self *StateTransition) TransitionDb() (ret []byte, requiredGas, usedGas *big.Int, err error) { if err = self.preCheck(); err != nil { return @@ -217,26 +222,32 @@ func (self *StateTransition) TransitionDb() (ret []byte, requiredGas, usedGas *b msg := self.msg sender := self.from() // err checked in preCheck - homestead := self.env.ChainConfig().IsHomestead(self.env.BlockNumber) + homestead := self.evm.ChainConfig().IsHomestead(self.evm.BlockNumber) contractCreation := MessageCreatesContract(msg) // Pay intrinsic gas - if err = self.useGas(IntrinsicGas(self.data, contractCreation, homestead)); err != nil { + // TODO convert to uint64 + intrinsicGas := IntrinsicGas(self.data, contractCreation, homestead) + if intrinsicGas.BitLen() > 64 { + return nil, nil, nil, InvalidTxError(vm.ErrOutOfGas) + } + + if err = self.useGas(intrinsicGas.Uint64()); err != nil { return nil, nil, nil, InvalidTxError(err) } var ( - vmenv = self.env + evm = self.evm // vm errors do not effect consensus and are therefor // not assigned to err, except for insufficient balance // error. vmerr error ) if contractCreation { - ret, _, vmerr = vmenv.Create(sender, self.data, self.gas, self.value) + ret, _, self.gas, vmerr = evm.Create(sender, self.data, self.gas, self.value) } else { // Increment the nonce for the next transaction self.state.SetNonce(sender.Address(), self.state.GetNonce(sender.Address())+1) - ret, vmerr = vmenv.Call(sender, self.to().Address(), self.data, self.gas, self.value) + ret, self.gas, vmerr = evm.Call(sender, self.to().Address(), self.data, self.gas, self.value) } if vmerr != nil { glog.V(logger.Core).Infoln("vm returned with error:", err) @@ -251,7 +262,7 @@ func (self *StateTransition) TransitionDb() (ret []byte, requiredGas, usedGas *b requiredGas = new(big.Int).Set(self.gasUsed()) self.refundGas() - self.state.AddBalance(self.env.Coinbase, new(big.Int).Mul(self.gasUsed(), self.gasPrice)) + self.state.AddBalance(self.evm.Coinbase, new(big.Int).Mul(self.gasUsed(), self.gasPrice)) return ret, requiredGas, self.gasUsed(), err } @@ -260,20 +271,21 @@ func (self *StateTransition) refundGas() { // Return eth for remaining gas to the sender account, // exchanged at the original rate. sender := self.from() // err already checked - remaining := new(big.Int).Mul(self.gas, self.gasPrice) + remaining := new(big.Int).Mul(new(big.Int).SetUint64(self.gas), self.gasPrice) sender.AddBalance(remaining) // Apply refund counter, capped to half of the used gas. uhalf := remaining.Div(self.gasUsed(), common.Big2) refund := common.BigMin(uhalf, self.state.GetRefund()) - self.gas.Add(self.gas, refund) + self.gas += refund.Uint64() + self.state.AddBalance(sender.Address(), refund.Mul(refund, self.gasPrice)) // Also return remaining gas to the block gas counter so it is // available for the next transaction. - self.gp.AddGas(self.gas) + self.gp.AddGas(new(big.Int).SetUint64(self.gas)) } func (self *StateTransition) gasUsed() *big.Int { - return new(big.Int).Sub(self.initialGas, self.gas) + return new(big.Int).Sub(self.initialGas, new(big.Int).SetUint64(self.gas)) } diff --git a/core/vm/common.go b/core/vm/common.go index 2878b92d2a..b7b9a6a9c7 100644 --- a/core/vm/common.go +++ b/core/vm/common.go @@ -21,28 +21,11 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/params" -) - -// Type is the VM type accepted by **NewVm** -type Type byte - -const ( - StdVmTy Type = iota // Default standard VM - JitVmTy // LLVM JIT VM - MaxVmTy ) var ( - Pow256 = common.BigPow(2, 256) // Pow256 is 2**256 - U256 = common.U256 // Shortcut to common.U256 S256 = common.S256 // Shortcut to common.S256 - - Zero = common.Big0 // Shortcut to common.Big0 - One = common.Big1 // Shortcut to common.Big1 - - max = big.NewInt(math.MaxInt64) // Maximum 64 bit integer ) // calculates the memory size required for a step @@ -54,48 +37,6 @@ func calcMemSize(off, l *big.Int) *big.Int { return new(big.Int).Add(off, l) } -// calculates the quadratic gas -func quadMemGas(mem *Memory, newMemSize, gas *big.Int) { - if newMemSize.Cmp(common.Big0) > 0 { - newMemSizeWords := toWordSize(newMemSize) - newMemSize.Mul(newMemSizeWords, u256(32)) - - if newMemSize.Cmp(u256(int64(mem.Len()))) > 0 { - // be careful reusing variables here when changing. - // The order has been optimised to reduce allocation - oldSize := toWordSize(big.NewInt(int64(mem.Len()))) - pow := new(big.Int).Exp(oldSize, common.Big2, Zero) - linCoef := oldSize.Mul(oldSize, params.MemoryGas) - quadCoef := new(big.Int).Div(pow, params.QuadCoeffDiv) - oldTotalFee := new(big.Int).Add(linCoef, quadCoef) - - pow.Exp(newMemSizeWords, common.Big2, Zero) - linCoef = linCoef.Mul(newMemSizeWords, params.MemoryGas) - quadCoef = quadCoef.Div(pow, params.QuadCoeffDiv) - newTotalFee := linCoef.Add(linCoef, quadCoef) - - fee := newTotalFee.Sub(newTotalFee, oldTotalFee) - gas.Add(gas, fee) - } - } -} - -// Simple helper -func u256(n int64) *big.Int { - return big.NewInt(n) -} - -// Mainly used for print variables and passing to Print* -func toValue(val *big.Int) interface{} { - // Let's assume a string on right padded zero's - b := val.Bytes() - if b[0] != 0 && b[len(b)-1] == 0x0 && b[len(b)-2] == 0x0 { - return string(b) - } - - return val -} - // getData returns a slice from the data based on the start and size and pads // up to size with zero's. This function is overflow safe. func getData(data []byte, start, size *big.Int) []byte { @@ -106,14 +47,17 @@ func getData(data []byte, start, size *big.Int) []byte { return common.RightPadBytes(data[s.Uint64():e.Uint64()], int(size.Uint64())) } -// useGas attempts to subtract the amount of gas and returns whether it was -// successful -func useGas(gas, amount *big.Int) bool { - if gas.Cmp(amount) < 0 { - return false +// bigUint64 returns the integer casted to a uint64 and returns whether it +// overflowed in the process. +func bigUint64(v *big.Int) (uint64, bool) { + return v.Uint64(), v.BitLen() > 64 +} + +// toWordSize returns the ceiled word size required for memory expansion. +func toWordSize(size uint64) uint64 { + if size > math.MaxUint64-31 { + return math.MaxUint64/32 + 1 } - // Sub the amount of gas from the remaining - gas.Sub(gas, amount) - return true + return (size + 31) / 32 } diff --git a/core/vm/contract.go b/core/vm/contract.go index dfa93ab18b..091106d84d 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -24,7 +24,6 @@ import ( // ContractRef is a reference to the contract's backing object type ContractRef interface { - ReturnGas(*big.Int) Address() common.Address Value() *big.Int SetCode(common.Hash, []byte) @@ -48,7 +47,8 @@ type Contract struct { CodeAddr *common.Address Input []byte - value, Gas, UsedGas *big.Int + Gas uint64 + value *big.Int Args []byte @@ -56,7 +56,7 @@ type Contract struct { } // NewContract returns a new contract environment for the execution of EVM. -func NewContract(caller ContractRef, object ContractRef, value, gas *big.Int) *Contract { +func NewContract(caller ContractRef, object ContractRef, value *big.Int, gas uint64) *Contract { c := &Contract{CallerAddress: caller.Address(), caller: caller, self: object, Args: nil} if parent, ok := caller.(*Contract); ok { @@ -68,9 +68,8 @@ func NewContract(caller ContractRef, object ContractRef, value, gas *big.Int) *C // Gas should be a pointer so it can safely be reduced through the run // This pointer will be off the state transition - c.Gas = gas //new(big.Int).Set(gas) + c.Gas = gas c.value = new(big.Int).Set(value) - c.UsedGas = new(big.Int) return c } @@ -107,27 +106,13 @@ func (c *Contract) Caller() common.Address { return c.CallerAddress } -// Finalise finalises the contract and returning any remaining gas to the original -// caller. -func (c *Contract) Finalise() { - // Return the remaining gas to the caller - c.caller.ReturnGas(c.Gas) -} - // UseGas attempts the use gas and subtracts it and returns true on success -func (c *Contract) UseGas(gas *big.Int) (ok bool) { - ok = useGas(c.Gas, gas) - if ok { - c.UsedGas.Add(c.UsedGas, gas) +func (c *Contract) UseGas(gas uint64) (ok bool) { + if c.Gas < gas { + return false } - return -} - -// ReturnGas adds the given gas back to itself. -func (c *Contract) ReturnGas(gas *big.Int) { - // Return the gas to the context - c.Gas.Add(c.Gas, gas) - c.UsedGas.Sub(c.UsedGas, gas) + c.Gas -= gas + return true } // Address returns the contracts address diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 9645d268fe..593b6ca55c 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -17,8 +17,6 @@ package vm import ( - "math/big" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/logger" @@ -30,8 +28,8 @@ import ( // requires a deterministic gas count based on the input size of the Run method of the // contract. type PrecompiledContract interface { - RequiredGas(inputSize int) *big.Int // RequiredPrice calculates the contract gas use - Run(input []byte) []byte // Run runs the precompiled contract + RequiredGas(inputSize int) uint64 // RequiredPrice calculates the contract gas use + Run(input []byte) []byte // Run runs the precompiled contract } // Precompiled contains the default set of ethereum contracts @@ -57,7 +55,7 @@ func RunPrecompiledContract(p PrecompiledContract, input []byte, contract *Contr // ECRECOVER implemented as a native contract type ecrecover struct{} -func (c *ecrecover) RequiredGas(inputSize int) *big.Int { +func (c *ecrecover) RequiredGas(inputSize int) uint64 { return params.EcrecoverGas } @@ -92,10 +90,12 @@ func (c *ecrecover) Run(in []byte) []byte { // SHA256 implemented as a native contract type sha256 struct{} -func (c *sha256) RequiredGas(inputSize int) *big.Int { - n := big.NewInt(int64(inputSize+31) / 32) - n.Mul(n, params.Sha256WordGas) - return n.Add(n, params.Sha256Gas) +// RequiredGas returns the gas required to execute the pre-compiled contract. +// +// This method does not require any overflow checking as the input size gas costs +// required for anything significant is so high it's impossible to pay for. +func (c *sha256) RequiredGas(inputSize int) uint64 { + return uint64(inputSize+31)/32*params.Sha256WordGas + params.Sha256Gas } func (c *sha256) Run(in []byte) []byte { return crypto.Sha256(in) @@ -104,10 +104,12 @@ func (c *sha256) Run(in []byte) []byte { // RIPMED160 implemented as a native contract type ripemd160 struct{} -func (c *ripemd160) RequiredGas(inputSize int) *big.Int { - n := big.NewInt(int64(inputSize+31) / 32) - n.Mul(n, params.Ripemd160WordGas) - return n.Add(n, params.Ripemd160Gas) +// RequiredGas returns the gas required to execute the pre-compiled contract. +// +// This method does not require any overflow checking as the input size gas costs +// required for anything significant is so high it's impossible to pay for. +func (c *ripemd160) RequiredGas(inputSize int) uint64 { + return uint64(inputSize+31)/32*params.Ripemd160WordGas + params.Ripemd160Gas } func (c *ripemd160) Run(in []byte) []byte { return common.LeftPadBytes(crypto.Ripemd160(in), 32) @@ -116,11 +118,12 @@ func (c *ripemd160) Run(in []byte) []byte { // data copy implemented as a native contract type dataCopy struct{} -func (c *dataCopy) RequiredGas(inputSize int) *big.Int { - n := big.NewInt(int64(inputSize+31) / 32) - n.Mul(n, params.IdentityWordGas) - - return n.Add(n, params.IdentityGas) +// RequiredGas returns the gas required to execute the pre-compiled contract. +// +// This method does not require any overflow checking as the input size gas costs +// required for anything significant is so high it's impossible to pay for. +func (c *dataCopy) RequiredGas(inputSize int) uint64 { + return uint64(inputSize+31)/32*params.IdentityWordGas + params.IdentityGas } func (c *dataCopy) Run(in []byte) []byte { return in diff --git a/core/vm/environment.go b/core/vm/evm.go similarity index 85% rename from core/vm/environment.go rename to core/vm/evm.go index c19ef464ba..0c5d998c29 100644 --- a/core/vm/environment.go +++ b/core/vm/evm.go @@ -17,7 +17,6 @@ package vm import ( - "fmt" "math/big" "sync/atomic" @@ -102,24 +101,18 @@ func (evm *EVM) Cancel() { // Call executes the contract associated with the addr with the given input as parameters. It also handles any // necessary value transfer required and takes the necessary steps to create accounts and reverses the state in // case of an execution error or failed value transfer. -func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas, value *big.Int) (ret []byte, err error) { +func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas uint64, value *big.Int) (ret []byte, leftOverGas uint64, err error) { if evm.vmConfig.NoRecursion && evm.depth > 0 { - caller.ReturnGas(gas) - - return nil, nil + return nil, gas, nil } // Depth check execution. Fail if we're trying to execute above the // limit. - if evm.depth > int(params.CallCreateDepth.Int64()) { - caller.ReturnGas(gas) - - return nil, ErrDepth + if evm.depth > int(params.CallCreateDepth) { + return nil, gas, ErrDepth } if !evm.Context.CanTransfer(evm.StateDB, caller.Address(), value) { - caller.ReturnGas(gas) - - return nil, ErrInsufficientBalance + return nil, gas, ErrInsufficientBalance } var ( @@ -128,8 +121,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas, ) if !evm.StateDB.Exist(addr) { if PrecompiledContracts[addr] == nil && evm.ChainConfig().IsEIP158(evm.BlockNumber) && value.BitLen() == 0 { - caller.ReturnGas(gas) - return nil, nil + return nil, gas, nil } to = evm.StateDB.CreateAccount(addr) @@ -143,7 +135,6 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas, // only. contract := NewContract(caller, to, value, gas) contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) - defer contract.Finalise() ret, err = evm.interpreter.Run(contract, input) // When an error was returned by the EVM or when setting the creation code @@ -154,7 +145,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas, evm.StateDB.RevertToSnapshot(snapshot) } - return ret, err + return ret, contract.Gas, err } // CallCode executes the contract associated with the addr with the given input as parameters. It also handles any @@ -162,24 +153,18 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas, // case of an execution error or failed value transfer. // // CallCode differs from Call in the sense that it executes the given address' code with the caller as context. -func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, gas, value *big.Int) (ret []byte, err error) { +func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, gas uint64, value *big.Int) (ret []byte, leftOverGas uint64, err error) { if evm.vmConfig.NoRecursion && evm.depth > 0 { - caller.ReturnGas(gas) - - return nil, nil + return nil, gas, nil } // Depth check execution. Fail if we're trying to execute above the // limit. - if evm.depth > int(params.CallCreateDepth.Int64()) { - caller.ReturnGas(gas) - - return nil, ErrDepth + if evm.depth > int(params.CallCreateDepth) { + return nil, gas, ErrDepth } if !evm.CanTransfer(evm.StateDB, caller.Address(), value) { - caller.ReturnGas(gas) - - return nil, fmt.Errorf("insufficient funds to transfer value. Req %v, has %v", value, evm.StateDB.GetBalance(caller.Address())) + return nil, gas, ErrInsufficientBalance } var ( @@ -191,7 +176,6 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, // only. contract := NewContract(caller, to, value, gas) contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) - defer contract.Finalise() ret, err = evm.interpreter.Run(contract, input) if err != nil { @@ -200,7 +184,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, evm.StateDB.RevertToSnapshot(snapshot) } - return ret, err + return ret, contract.Gas, err } // DelegateCall executes the contract associated with the addr with the given input as parameters. @@ -208,18 +192,15 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, // // DelegateCall differs from CallCode in the sense that it executes the given address' code with the caller as context // and the caller is set to the caller of the caller. -func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []byte, gas *big.Int) (ret []byte, err error) { +func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []byte, gas uint64) (ret []byte, leftOverGas uint64, err error) { if evm.vmConfig.NoRecursion && evm.depth > 0 { - caller.ReturnGas(gas) - - return nil, nil + return nil, gas, nil } // Depth check execution. Fail if we're trying to execute above the // limit. - if evm.depth > int(params.CallCreateDepth.Int64()) { - caller.ReturnGas(gas) - return nil, ErrDepth + if evm.depth > int(params.CallCreateDepth) { + return nil, gas, ErrDepth } var ( @@ -230,7 +211,6 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by // Iinitialise a new contract and make initialise the delegate values contract := NewContract(caller, to, caller.Value(), gas).AsDelegate() contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) - defer contract.Finalise() ret, err = evm.interpreter.Run(contract, input) if err != nil { @@ -239,28 +219,22 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by evm.StateDB.RevertToSnapshot(snapshot) } - return ret, err + return ret, contract.Gas, err } // Create creates a new contract using code as deployment code. -func (evm *EVM) Create(caller ContractRef, code []byte, gas, value *big.Int) (ret []byte, contractAddr common.Address, err error) { +func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { if evm.vmConfig.NoRecursion && evm.depth > 0 { - caller.ReturnGas(gas) - - return nil, common.Address{}, nil + return nil, common.Address{}, gas, nil } // Depth check execution. Fail if we're trying to execute above the // limit. - if evm.depth > int(params.CallCreateDepth.Int64()) { - caller.ReturnGas(gas) - - return nil, common.Address{}, ErrDepth + if evm.depth > int(params.CallCreateDepth) { + return nil, common.Address{}, gas, ErrDepth } if !evm.CanTransfer(evm.StateDB, caller.Address(), value) { - caller.ReturnGas(gas) - - return nil, common.Address{}, ErrInsufficientBalance + return nil, common.Address{}, gas, ErrInsufficientBalance } // Create a new account on the state @@ -280,7 +254,6 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas, value *big.Int) (re // only. contract := NewContract(caller, to, value, gas) contract.SetCallCode(&contractAddr, crypto.Keccak256Hash(code), code) - defer contract.Finalise() ret, err = evm.interpreter.Run(contract, nil) @@ -291,9 +264,8 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas, value *big.Int) (re // be stored due to not enough gas set an error and let it be handled // by the error checking condition below. if err == nil && !maxCodeSizeExceeded { - dataGas := big.NewInt(int64(len(ret))) - dataGas.Mul(dataGas, params.CreateDataGas) - if contract.UseGas(dataGas) { + createDataGas := uint64(len(ret)) * params.CreateDataGas + if contract.UseGas(createDataGas) { evm.StateDB.SetCode(contractAddr, ret) } else { err = ErrCodeStoreOutOfGas @@ -305,11 +277,10 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas, value *big.Int) (re // when we're in homestead this also counts for code storage gas errors. if maxCodeSizeExceeded || (err != nil && (evm.ChainConfig().IsHomestead(evm.BlockNumber) || err != ErrCodeStoreOutOfGas)) { - contract.UseGas(contract.Gas) evm.StateDB.RevertToSnapshot(snapshot) // Nothing should be returned when an error is thrown. - return nil, contractAddr, err + return nil, contractAddr, 0, err } // If the vm returned with an error the return value should be set to nil. // This isn't consensus critical but merely to for behaviour reasons such as @@ -318,7 +289,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas, value *big.Int) (re ret = nil } - return ret, contractAddr, err + return ret, contractAddr, contract.Gas, err } // ChainConfig returns the evmironment's chain configuration diff --git a/core/vm/gas.go b/core/vm/gas.go index fdbc0df7f7..dd64d5f178 100644 --- a/core/vm/gas.go +++ b/core/vm/gas.go @@ -17,149 +17,42 @@ package vm import ( - "fmt" "math/big" "github.com/ethereum/go-ethereum/params" ) -var ( - GasQuickStep = big.NewInt(2) - GasFastestStep = big.NewInt(3) - GasFastStep = big.NewInt(5) - GasMidStep = big.NewInt(8) - GasSlowStep = big.NewInt(10) - GasExtStep = big.NewInt(20) +const ( + GasQuickStep uint64 = 2 + GasFastestStep uint64 = 3 + GasFastStep uint64 = 5 + GasMidStep uint64 = 8 + GasSlowStep uint64 = 10 + GasExtStep uint64 = 20 - GasReturn = big.NewInt(0) - GasStop = big.NewInt(0) - - GasContractByte = big.NewInt(200) - - n64 = big.NewInt(64) + GasReturn uint64 = 0 + GasStop uint64 = 0 + GasContractByte uint64 = 200 ) // calcGas returns the actual gas cost of the call. // // The cost of gas was changed during the homestead price change HF. To allow for EIP150 // to be implemented. The returned gas is gas - base * 63 / 64. -func callGas(gasTable params.GasTable, availableGas, base, callCost *big.Int) *big.Int { - if gasTable.CreateBySuicide != nil { - availableGas = new(big.Int).Sub(availableGas, base) - g := new(big.Int).Div(availableGas, n64) - g.Sub(availableGas, g) - - if g.Cmp(callCost) < 0 { - return g +func callGas(gasTable params.GasTable, availableGas, base uint64, callCost *big.Int) (uint64, error) { + if gasTable.CreateBySuicide > 0 { + availableGas = availableGas - base + gas := availableGas - availableGas/64 + // If the bit length exceeds 64 bit we know that the newly calculated "gas" for EIP150 + // is smaller than the requested amount. Therefor we return the new gas instead + // of returning an error. + if callCost.BitLen() > 64 || gas < callCost.Uint64() { + return gas, nil } } - return callCost -} - -// baseCheck checks for any stack error underflows -func baseCheck(op OpCode, stack *Stack, gas *big.Int) error { - // PUSH and DUP are a bit special. They all cost the same but we do want to have checking on stack push limit - // PUSH is also allowed to calculate the same price for all PUSHes - // DUP requirements are handled elsewhere (except for the stack limit check) - if op >= PUSH1 && op <= PUSH32 { - op = PUSH1 - } - if op >= DUP1 && op <= DUP16 { - op = DUP1 + if callCost.BitLen() > 64 { + return 0, errGasUintOverflow } - if r, ok := _baseCheck[op]; ok { - err := stack.require(r.stackPop) - if err != nil { - return err - } - - if r.stackPush > 0 && stack.len()-r.stackPop+r.stackPush > int(params.StackLimit.Int64()) { - return fmt.Errorf("stack limit reached %d (%d)", stack.len(), params.StackLimit.Int64()) - } - - gas.Add(gas, r.gas) - } - return nil -} - -// casts a arbitrary number to the amount of words (sets of 32 bytes) -func toWordSize(size *big.Int) *big.Int { - tmp := new(big.Int) - tmp.Add(size, u256(31)) - tmp.Div(tmp, u256(32)) - return tmp -} - -type req struct { - stackPop int - gas *big.Int - stackPush int -} - -var _baseCheck = map[OpCode]req{ - // opcode | stack pop | gas price | stack push - ADD: {2, GasFastestStep, 1}, - LT: {2, GasFastestStep, 1}, - GT: {2, GasFastestStep, 1}, - SLT: {2, GasFastestStep, 1}, - SGT: {2, GasFastestStep, 1}, - EQ: {2, GasFastestStep, 1}, - ISZERO: {1, GasFastestStep, 1}, - SUB: {2, GasFastestStep, 1}, - AND: {2, GasFastestStep, 1}, - OR: {2, GasFastestStep, 1}, - XOR: {2, GasFastestStep, 1}, - NOT: {1, GasFastestStep, 1}, - BYTE: {2, GasFastestStep, 1}, - CALLDATALOAD: {1, GasFastestStep, 1}, - CALLDATACOPY: {3, GasFastestStep, 1}, - MLOAD: {1, GasFastestStep, 1}, - MSTORE: {2, GasFastestStep, 0}, - MSTORE8: {2, GasFastestStep, 0}, - CODECOPY: {3, GasFastestStep, 0}, - MUL: {2, GasFastStep, 1}, - DIV: {2, GasFastStep, 1}, - SDIV: {2, GasFastStep, 1}, - MOD: {2, GasFastStep, 1}, - SMOD: {2, GasFastStep, 1}, - SIGNEXTEND: {2, GasFastStep, 1}, - ADDMOD: {3, GasMidStep, 1}, - MULMOD: {3, GasMidStep, 1}, - JUMP: {1, GasMidStep, 0}, - JUMPI: {2, GasSlowStep, 0}, - EXP: {2, GasSlowStep, 1}, - ADDRESS: {0, GasQuickStep, 1}, - ORIGIN: {0, GasQuickStep, 1}, - CALLER: {0, GasQuickStep, 1}, - CALLVALUE: {0, GasQuickStep, 1}, - CODESIZE: {0, GasQuickStep, 1}, - GASPRICE: {0, GasQuickStep, 1}, - COINBASE: {0, GasQuickStep, 1}, - TIMESTAMP: {0, GasQuickStep, 1}, - NUMBER: {0, GasQuickStep, 1}, - CALLDATASIZE: {0, GasQuickStep, 1}, - DIFFICULTY: {0, GasQuickStep, 1}, - GASLIMIT: {0, GasQuickStep, 1}, - POP: {1, GasQuickStep, 0}, - PC: {0, GasQuickStep, 1}, - MSIZE: {0, GasQuickStep, 1}, - GAS: {0, GasQuickStep, 1}, - BLOCKHASH: {1, GasExtStep, 1}, - BALANCE: {1, Zero, 1}, - EXTCODESIZE: {1, Zero, 1}, - EXTCODECOPY: {4, Zero, 0}, - SLOAD: {1, params.SloadGas, 1}, - SSTORE: {2, Zero, 0}, - SHA3: {2, params.Sha3Gas, 1}, - CREATE: {3, params.CreateGas, 1}, - // Zero is calculated in the gasSwitch - CALL: {7, Zero, 1}, - CALLCODE: {7, Zero, 1}, - DELEGATECALL: {6, Zero, 1}, - SUICIDE: {1, Zero, 0}, - JUMPDEST: {0, params.JumpdestGas, 0}, - RETURN: {2, Zero, 0}, - PUSH1: {0, GasFastestStep, 1}, - DUP1: {0, Zero, 1}, + return callCost.Uint64(), nil } diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 4d2c4e94ae..fba1eb066e 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -1,56 +1,80 @@ package vm import ( + gmath "math" "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/params" ) -func memoryGasCost(mem *Memory, newMemSize *big.Int) *big.Int { - gas := new(big.Int) - if newMemSize.Cmp(common.Big0) > 0 { - newMemSizeWords := toWordSize(newMemSize) - - if newMemSize.Cmp(u256(int64(mem.Len()))) > 0 { - // be careful reusing variables here when changing. - // The order has been optimised to reduce allocation - oldSize := toWordSize(big.NewInt(int64(mem.Len()))) - pow := new(big.Int).Exp(oldSize, common.Big2, Zero) - linCoef := oldSize.Mul(oldSize, params.MemoryGas) - quadCoef := new(big.Int).Div(pow, params.QuadCoeffDiv) - oldTotalFee := new(big.Int).Add(linCoef, quadCoef) - - pow.Exp(newMemSizeWords, common.Big2, Zero) - linCoef = linCoef.Mul(newMemSizeWords, params.MemoryGas) - quadCoef = quadCoef.Div(pow, params.QuadCoeffDiv) - newTotalFee := linCoef.Add(linCoef, quadCoef) - - fee := newTotalFee.Sub(newTotalFee, oldTotalFee) - gas.Add(gas, fee) - } +// memoryGasCosts calculates the quadratic gas for memory expansion. It does so +// only for the memory region that is expanded, not the total memory. +func memoryGasCost(mem *Memory, newMemSize uint64) (uint64, error) { + // The maximum that will fit in a uint64 is max_word_count - 1 + // anything above that will result in an overflow. + if newMemSize > gmath.MaxUint64-32 { + return 0, errGasUintOverflow } - return gas + + if newMemSize == 0 { + return 0, nil + } + + newMemSizeWords := toWordSize(newMemSize) + newMemSize = newMemSizeWords * 32 + + if newMemSize > uint64(mem.Len()) { + square := newMemSizeWords * newMemSizeWords + linCoef := newMemSizeWords * params.MemoryGas + quadCoef := square / params.QuadCoeffDiv + newTotalFee := linCoef + quadCoef + + fee := newTotalFee - mem.lastGasCost + mem.lastGasCost = newTotalFee + + return fee, nil + } + return 0, nil } -func constGasFunc(gas *big.Int) gasFunc { - return func(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return gas +func constGasFunc(gas uint64) gasFunc { + return func(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + return gas, nil } } -func gasCalldataCopy(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - gas := memoryGasCost(mem, memorySize) - gas.Add(gas, GasFastestStep) - words := toWordSize(stack.Back(2)) +func gasCalldataCopy(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + gas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, err + } - return gas.Add(gas, words.Mul(words, params.CopyGas)) + var overflow bool + if gas, overflow = math.SafeAdd(gas, GasFastestStep); overflow { + return 0, errGasUintOverflow + } + + words, overflow := bigUint64(stack.Back(2)) + if overflow { + return 0, errGasUintOverflow + } + + if words, overflow = math.SafeMul(toWordSize(words), params.CopyGas); overflow { + return 0, errGasUintOverflow + } + + if gas, overflow = math.SafeAdd(gas, words); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasSStore(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { +func gasSStore(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { var ( y, x = stack.Back(1), stack.Back(0) - val = env.StateDB.GetState(contract.Address(), common.BigToHash(x)) + val = evm.StateDB.GetState(contract.Address(), common.BigToHash(x)) ) // This checks for 3 scenario's and calculates gas accordingly // 1. From a zero-value address to a non-zero value (NEW VALUE) @@ -58,189 +82,335 @@ func gasSStore(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, m // 3. From a non-zero to a non-zero (CHANGE) if common.EmptyHash(val) && !common.EmptyHash(common.BigToHash(y)) { // 0 => non 0 - return new(big.Int).Set(params.SstoreSetGas) + return params.SstoreSetGas, nil } else if !common.EmptyHash(val) && common.EmptyHash(common.BigToHash(y)) { - env.StateDB.AddRefund(params.SstoreRefundGas) + evm.StateDB.AddRefund(new(big.Int).SetUint64(params.SstoreRefundGas)) - return new(big.Int).Set(params.SstoreClearGas) + return params.SstoreClearGas, nil } else { // non 0 => non 0 (or 0 => 0) - return new(big.Int).Set(params.SstoreResetGas) + return params.SstoreResetGas, nil } } -func makeGasLog(n uint) gasFunc { - return func(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - mSize := stack.Back(1) +func makeGasLog(n uint64) gasFunc { + return func(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + requestedSize, overflow := bigUint64(stack.Back(1)) + if overflow { + return 0, errGasUintOverflow + } - gas := new(big.Int).Add(memoryGasCost(mem, memorySize), params.LogGas) - gas.Add(gas, new(big.Int).Mul(big.NewInt(int64(n)), params.LogTopicGas)) - gas.Add(gas, new(big.Int).Mul(mSize, params.LogDataGas)) - return gas + gas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, err + } + + if gas, overflow = math.SafeAdd(gas, params.LogGas); overflow { + return 0, errGasUintOverflow + } + if gas, overflow = math.SafeAdd(gas, n*params.LogTopicGas); overflow { + return 0, errGasUintOverflow + } + + var memorySizeGas uint64 + if memorySizeGas, overflow = math.SafeMul(requestedSize, params.LogDataGas); overflow { + return 0, errGasUintOverflow + } + if gas, overflow = math.SafeAdd(gas, memorySizeGas); overflow { + return 0, errGasUintOverflow + } + return gas, nil } } -func gasSha3(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - gas := memoryGasCost(mem, memorySize) - gas.Add(gas, params.Sha3Gas) - words := toWordSize(stack.Back(1)) - return gas.Add(gas, words.Mul(words, params.Sha3WordGas)) +func gasSha3(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + var overflow bool + gas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, err + } + + if gas, overflow = math.SafeAdd(gas, params.Sha3Gas); overflow { + return 0, errGasUintOverflow + } + + wordGas, overflow := bigUint64(stack.Back(1)) + if overflow { + return 0, errGasUintOverflow + } + if wordGas, overflow = math.SafeMul(toWordSize(wordGas), params.Sha3WordGas); overflow { + return 0, errGasUintOverflow + } + if gas, overflow = math.SafeAdd(gas, wordGas); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasCodeCopy(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - gas := memoryGasCost(mem, memorySize) - gas.Add(gas, GasFastestStep) - words := toWordSize(stack.Back(2)) +func gasCodeCopy(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + gas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, err + } - return gas.Add(gas, words.Mul(words, params.CopyGas)) + var overflow bool + if gas, overflow = math.SafeAdd(gas, GasFastestStep); overflow { + return 0, errGasUintOverflow + } + + wordGas, overflow := bigUint64(stack.Back(2)) + if overflow { + return 0, errGasUintOverflow + } + if wordGas, overflow = math.SafeMul(toWordSize(wordGas), params.CopyGas); overflow { + return 0, errGasUintOverflow + } + if gas, overflow = math.SafeAdd(gas, wordGas); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasExtCodeCopy(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - gas := memoryGasCost(mem, memorySize) - gas.Add(gas, gt.ExtcodeCopy) - words := toWordSize(stack.Back(3)) +func gasExtCodeCopy(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + gas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, err + } - return gas.Add(gas, words.Mul(words, params.CopyGas)) + var overflow bool + if gas, overflow = math.SafeAdd(gas, gt.ExtcodeCopy); overflow { + return 0, errGasUintOverflow + } + + wordGas, overflow := bigUint64(stack.Back(3)) + if overflow { + return 0, errGasUintOverflow + } + + if wordGas, overflow = math.SafeMul(toWordSize(wordGas), params.CopyGas); overflow { + return 0, errGasUintOverflow + } + + if gas, overflow = math.SafeAdd(gas, wordGas); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasMLoad(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return new(big.Int).Add(GasFastestStep, memoryGasCost(mem, memorySize)) +func gasMLoad(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + var overflow bool + gas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, errGasUintOverflow + } + if gas, overflow = math.SafeAdd(gas, GasFastestStep); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasMStore8(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return new(big.Int).Add(GasFastestStep, memoryGasCost(mem, memorySize)) +func gasMStore8(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + var overflow bool + gas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, errGasUintOverflow + } + if gas, overflow = math.SafeAdd(gas, GasFastestStep); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasMStore(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return new(big.Int).Add(GasFastestStep, memoryGasCost(mem, memorySize)) +func gasMStore(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + var overflow bool + gas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, errGasUintOverflow + } + if gas, overflow = math.SafeAdd(gas, GasFastestStep); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasCreate(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return new(big.Int).Add(params.CreateGas, memoryGasCost(mem, memorySize)) +func gasCreate(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + var overflow bool + gas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, err + } + if gas, overflow = math.SafeAdd(gas, params.CreateGas); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasBalance(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return gt.Balance +func gasBalance(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + return gt.Balance, nil } -func gasExtCodeSize(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return gt.ExtcodeSize +func gasExtCodeSize(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + return gt.ExtcodeSize, nil } -func gasSLoad(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return gt.SLoad +func gasSLoad(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + return gt.SLoad, nil } -func gasExp(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - expByteLen := int64((stack.data[stack.len()-2].BitLen() + 7) / 8) - gas := big.NewInt(expByteLen) - gas.Mul(gas, gt.ExpByte) - return gas.Add(gas, GasSlowStep) -} +func gasExp(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + expByteLen := uint64((stack.data[stack.len()-2].BitLen() + 7) / 8) -func gasCall(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - gas := new(big.Int).Set(gt.Calls) - - transfersValue := stack.Back(2).BitLen() > 0 var ( - address = common.BigToAddress(stack.Back(1)) - eip158 = env.ChainConfig().IsEIP158(env.BlockNumber) + gas = expByteLen * gt.ExpByte // no overflow check required. Max is 256 * ExpByte gas + overflow bool + ) + if gas, overflow = math.SafeAdd(gas, GasSlowStep); overflow { + return 0, errGasUintOverflow + } + return gas, nil +} + +func gasCall(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + var ( + gas = gt.Calls + transfersValue = stack.Back(2).BitLen() > 0 + address = common.BigToAddress(stack.Back(1)) + eip158 = evm.ChainConfig().IsEIP158(evm.BlockNumber) ) if eip158 { - if env.StateDB.Empty(address) && transfersValue { - gas.Add(gas, params.CallNewAccountGas) + if evm.StateDB.Empty(address) && transfersValue { + gas += params.CallNewAccountGas } - } else if !env.StateDB.Exist(address) { - gas.Add(gas, params.CallNewAccountGas) + } else if !evm.StateDB.Exist(address) { + gas += params.CallNewAccountGas } if transfersValue { - gas.Add(gas, params.CallValueTransferGas) + gas += params.CallValueTransferGas + } + memoryGas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, err + } + var overflow bool + if gas, overflow = math.SafeAdd(gas, memoryGas); overflow { + return 0, errGasUintOverflow } - gas.Add(gas, memoryGasCost(mem, memorySize)) - cg := callGas(gt, contract.Gas, gas, stack.data[stack.len()-1]) + cg, err := callGas(gt, contract.Gas, gas, stack.Back(0)) + if err != nil { + return 0, err + } // Replace the stack item with the new gas calculation. This means that // either the original item is left on the stack or the item is replaced by: // (availableGas - gas) * 63 / 64 // We replace the stack item so that it's available when the opCall instruction is // called. This information is otherwise lost due to the dependency on *current* // available gas. - stack.data[stack.len()-1] = cg + stack.data[stack.len()-1] = new(big.Int).SetUint64(cg) - return gas.Add(gas, cg) + if gas, overflow = math.SafeAdd(gas, cg); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasCallCode(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - gas := new(big.Int).Set(gt.Calls) +func gasCallCode(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + gas := gt.Calls if stack.Back(2).BitLen() > 0 { - gas.Add(gas, params.CallValueTransferGas) + gas += params.CallValueTransferGas + } + memoryGas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, err + } + var overflow bool + if gas, overflow = math.SafeAdd(gas, memoryGas); overflow { + return 0, errGasUintOverflow } - gas.Add(gas, memoryGasCost(mem, memorySize)) - cg := callGas(gt, contract.Gas, gas, stack.data[stack.len()-1]) + cg, err := callGas(gt, contract.Gas, gas, stack.Back(0)) + if err != nil { + return 0, err + } // Replace the stack item with the new gas calculation. This means that // either the original item is left on the stack or the item is replaced by: // (availableGas - gas) * 63 / 64 // We replace the stack item so that it's available when the opCall instruction is // called. This information is otherwise lost due to the dependency on *current* // available gas. - stack.data[stack.len()-1] = cg + stack.data[stack.len()-1] = new(big.Int).SetUint64(cg) - return gas.Add(gas, cg) + if gas, overflow = math.SafeAdd(gas, cg); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasReturn(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { +func gasReturn(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { return memoryGasCost(mem, memorySize) } -func gasSuicide(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - gas := new(big.Int) +func gasSuicide(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + var gas uint64 // EIP150 homestead gas reprice fork: - if env.ChainConfig().IsEIP150(env.BlockNumber) { - gas.Set(gt.Suicide) + if evm.ChainConfig().IsEIP150(evm.BlockNumber) { + gas = gt.Suicide var ( address = common.BigToAddress(stack.Back(0)) - eip158 = env.ChainConfig().IsEIP158(env.BlockNumber) + eip158 = evm.ChainConfig().IsEIP158(evm.BlockNumber) ) if eip158 { // if empty and transfers value - if env.StateDB.Empty(address) && env.StateDB.GetBalance(contract.Address()).BitLen() > 0 { - gas.Add(gas, gt.CreateBySuicide) + if evm.StateDB.Empty(address) && evm.StateDB.GetBalance(contract.Address()).BitLen() > 0 { + gas += gt.CreateBySuicide } - } else if !env.StateDB.Exist(address) { - gas.Add(gas, gt.CreateBySuicide) + } else if !evm.StateDB.Exist(address) { + gas += gt.CreateBySuicide } } - if !env.StateDB.HasSuicided(contract.Address()) { - env.StateDB.AddRefund(params.SuicideRefundGas) + if !evm.StateDB.HasSuicided(contract.Address()) { + evm.StateDB.AddRefund(new(big.Int).SetUint64(params.SuicideRefundGas)) } - return gas + return gas, nil } -func gasDelegateCall(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - gas := new(big.Int).Add(gt.Calls, memoryGasCost(mem, memorySize)) +func gasDelegateCall(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + gas, err := memoryGasCost(mem, memorySize) + if err != nil { + return 0, err + } + var overflow bool + if gas, overflow = math.SafeAdd(gas, gt.Calls); overflow { + return 0, errGasUintOverflow + } - cg := callGas(gt, contract.Gas, gas, stack.data[stack.len()-1]) + cg, err := callGas(gt, contract.Gas, gas, stack.Back(0)) + if err != nil { + return 0, err + } // Replace the stack item with the new gas calculation. This means that // either the original item is left on the stack or the item is replaced by: // (availableGas - gas) * 63 / 64 // We replace the stack item so that it's available when the opCall instruction is // called. - stack.data[stack.len()-1] = cg + stack.data[stack.len()-1] = new(big.Int).SetUint64(cg) - return gas.Add(gas, cg) + if gas, overflow = math.SafeAdd(gas, cg); overflow { + return 0, errGasUintOverflow + } + return gas, nil } -func gasPush(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return GasFastestStep +func gasPush(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + return GasFastestStep, nil } -func gasSwap(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return GasFastestStep +func gasSwap(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + return GasFastestStep, nil } -func gasDup(gt params.GasTable, env *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize *big.Int) *big.Int { - return GasFastestStep +func gasDup(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + return GasFastestStep, nil } diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go new file mode 100644 index 0000000000..cceb892859 --- /dev/null +++ b/core/vm/gas_table_test.go @@ -0,0 +1,24 @@ +package vm + +import ( + "math" + "testing" +) + +func TestMemoryGasCost(t *testing.T) { + size := uint64(math.MaxUint64 - 64) + _, err := memoryGasCost(&Memory{}, size) + if err != nil { + t.Error("didn't expect error:", err) + } + + _, err = memoryGasCost(&Memory{}, size+32) + if err != nil { + t.Error("didn't expect error:", err) + } + + _, err = memoryGasCost(&Memory{}, size+33) + if err == nil { + t.Error("expected error") + } +} diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 3b1b06cca1..39e5c05870 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -27,42 +27,56 @@ import ( "github.com/ethereum/go-ethereum/params" ) -func opAdd(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +var bigZero = new(big.Int) + +func opAdd(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() stack.push(U256(x.Add(x, y))) + + evm.interpreter.intPool.put(y) + return nil, nil } -func opSub(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opSub(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() stack.push(U256(x.Sub(x, y))) + + evm.interpreter.intPool.put(y) + return nil, nil } -func opMul(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opMul(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() stack.push(U256(x.Mul(x, y))) + + evm.interpreter.intPool.put(y) + return nil, nil } -func opDiv(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opDiv(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() if y.Cmp(common.Big0) != 0 { stack.push(U256(x.Div(x, y))) } else { stack.push(new(big.Int)) } + + evm.interpreter.intPool.put(y) + return nil, nil } -func opSdiv(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opSdiv(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := S256(stack.pop()), S256(stack.pop()) if y.Cmp(common.Big0) == 0 { stack.push(new(big.Int)) return nil, nil } else { n := new(big.Int) - if new(big.Int).Mul(x, y).Cmp(common.Big0) < 0 { + if evm.interpreter.intPool.get().Mul(x, y).Cmp(common.Big0) < 0 { n.SetInt64(-1) } else { n.SetInt64(1) @@ -73,20 +87,22 @@ func opSdiv(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Sta stack.push(U256(res)) } + evm.interpreter.intPool.put(y) return nil, nil } -func opMod(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opMod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() if y.Cmp(common.Big0) == 0 { stack.push(new(big.Int)) } else { stack.push(U256(x.Mod(x, y))) } + evm.interpreter.intPool.put(y) return nil, nil } -func opSmod(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opSmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := S256(stack.pop()), S256(stack.pop()) if y.Cmp(common.Big0) == 0 { @@ -104,16 +120,20 @@ func opSmod(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Sta stack.push(U256(res)) } + evm.interpreter.intPool.put(y) return nil, nil } -func opExp(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opExp(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { base, exponent := stack.pop(), stack.pop() stack.push(math.Exp(base, exponent)) + + evm.interpreter.intPool.put(base, exponent) + return nil, nil } -func opSignExtend(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opSignExtend(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { back := stack.pop() if back.Cmp(big.NewInt(31)) < 0 { bit := uint(back.Uint64()*8 + 7) @@ -128,198 +148,231 @@ func opSignExtend(pc *uint64, env *EVM, contract *Contract, memory *Memory, stac stack.push(U256(num)) } + + evm.interpreter.intPool.put(back) return nil, nil } -func opNot(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opNot(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x := stack.pop() stack.push(U256(x.Not(x))) return nil, nil } -func opLt(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opLt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() if x.Cmp(y) < 0 { - stack.push(big.NewInt(1)) + stack.push(evm.interpreter.intPool.get().SetUint64(1)) } else { stack.push(new(big.Int)) } + + evm.interpreter.intPool.put(x, y) return nil, nil } -func opGt(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opGt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() if x.Cmp(y) > 0 { - stack.push(big.NewInt(1)) + stack.push(evm.interpreter.intPool.get().SetUint64(1)) } else { stack.push(new(big.Int)) } + + evm.interpreter.intPool.put(x, y) return nil, nil } -func opSlt(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opSlt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := S256(stack.pop()), S256(stack.pop()) if x.Cmp(S256(y)) < 0 { - stack.push(big.NewInt(1)) + stack.push(evm.interpreter.intPool.get().SetUint64(1)) } else { stack.push(new(big.Int)) } + + evm.interpreter.intPool.put(x, y) return nil, nil } -func opSgt(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opSgt(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := S256(stack.pop()), S256(stack.pop()) if x.Cmp(y) > 0 { - stack.push(big.NewInt(1)) + stack.push(evm.interpreter.intPool.get().SetUint64(1)) } else { stack.push(new(big.Int)) } + + evm.interpreter.intPool.put(x, y) return nil, nil } -func opEq(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opEq(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() if x.Cmp(y) == 0 { - stack.push(big.NewInt(1)) + stack.push(evm.interpreter.intPool.get().SetUint64(1)) } else { stack.push(new(big.Int)) } + + evm.interpreter.intPool.put(x, y) return nil, nil } -func opIszero(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opIszero(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x := stack.pop() if x.Cmp(common.Big0) > 0 { stack.push(new(big.Int)) } else { - stack.push(big.NewInt(1)) + stack.push(evm.interpreter.intPool.get().SetUint64(1)) } + + evm.interpreter.intPool.put(x) return nil, nil } -func opAnd(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opAnd(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() stack.push(x.And(x, y)) + + evm.interpreter.intPool.put(y) return nil, nil } -func opOr(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opOr(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() stack.push(x.Or(x, y)) + + evm.interpreter.intPool.put(y) return nil, nil } -func opXor(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opXor(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y := stack.pop(), stack.pop() stack.push(x.Xor(x, y)) + + evm.interpreter.intPool.put(y) return nil, nil } -func opByte(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opByte(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { th, val := stack.pop(), stack.pop() if th.Cmp(big.NewInt(32)) < 0 { - byte := big.NewInt(int64(common.LeftPadBytes(val.Bytes(), 32)[th.Int64()])) + byte := evm.interpreter.intPool.get().SetInt64(int64(common.LeftPadBytes(val.Bytes(), 32)[th.Int64()])) stack.push(byte) } else { stack.push(new(big.Int)) } + + evm.interpreter.intPool.put(th, val) return nil, nil } -func opAddmod(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opAddmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y, z := stack.pop(), stack.pop(), stack.pop() - if z.Cmp(Zero) > 0 { + if z.Cmp(bigZero) > 0 { add := x.Add(x, y) add.Mod(add, z) stack.push(U256(add)) } else { stack.push(new(big.Int)) } + + evm.interpreter.intPool.put(y, z) return nil, nil } -func opMulmod(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opMulmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { x, y, z := stack.pop(), stack.pop(), stack.pop() - if z.Cmp(Zero) > 0 { + if z.Cmp(bigZero) > 0 { mul := x.Mul(x, y) mul.Mod(mul, z) stack.push(U256(mul)) } else { stack.push(new(big.Int)) } + + evm.interpreter.intPool.put(y, z) return nil, nil } -func opSha3(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opSha3(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { offset, size := stack.pop(), stack.pop() data := memory.Get(offset.Int64(), size.Int64()) hash := crypto.Keccak256(data) - if env.vmConfig.EnablePreimageRecording { - env.StateDB.AddPreimage(common.BytesToHash(hash), data) + if evm.vmConfig.EnablePreimageRecording { + evm.StateDB.AddPreimage(common.BytesToHash(hash), data) } stack.push(common.BytesToBig(hash)) + + evm.interpreter.intPool.put(offset, size) return nil, nil } -func opAddress(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opAddress(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { stack.push(common.Bytes2Big(contract.Address().Bytes())) return nil, nil } -func opBalance(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opBalance(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { addr := common.BigToAddress(stack.pop()) - balance := env.StateDB.GetBalance(addr) + balance := evm.StateDB.GetBalance(addr) stack.push(new(big.Int).Set(balance)) return nil, nil } -func opOrigin(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(env.Origin.Big()) +func opOrigin(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(evm.Origin.Big()) return nil, nil } -func opCaller(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opCaller(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { stack.push(contract.Caller().Big()) return nil, nil } -func opCallValue(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(new(big.Int).Set(contract.value)) +func opCallValue(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(evm.interpreter.intPool.get().Set(contract.value)) return nil, nil } -func opCalldataLoad(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opCalldataLoad(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { stack.push(common.Bytes2Big(getData(contract.Input, stack.pop(), common.Big32))) return nil, nil } -func opCalldataSize(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(big.NewInt(int64(len(contract.Input)))) +func opCalldataSize(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(evm.interpreter.intPool.get().SetInt64(int64(len(contract.Input)))) return nil, nil } -func opCalldataCopy(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opCalldataCopy(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { var ( mOff = stack.pop() cOff = stack.pop() l = stack.pop() ) memory.Set(mOff.Uint64(), l.Uint64(), getData(contract.Input, cOff, l)) + + evm.interpreter.intPool.put(mOff, cOff, l) return nil, nil } -func opExtCodeSize(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - addr := common.BigToAddress(stack.pop()) - l := big.NewInt(int64(env.StateDB.GetCodeSize(addr))) +func opExtCodeSize(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + a := stack.pop() + + addr := common.BigToAddress(a) + a.SetInt64(int64(evm.StateDB.GetCodeSize(addr))) + stack.push(a) + + return nil, nil +} + +func opCodeSize(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + l := evm.interpreter.intPool.get().SetInt64(int64(len(contract.Code))) stack.push(l) return nil, nil } -func opCodeSize(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - l := big.NewInt(int64(len(contract.Code))) - stack.push(l) - return nil, nil -} - -func opCodeCopy(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opCodeCopy(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { var ( mOff = stack.pop() cOff = stack.pop() @@ -328,113 +381,129 @@ func opCodeCopy(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack codeCopy := getData(contract.Code, cOff, l) memory.Set(mOff.Uint64(), l.Uint64(), codeCopy) + + evm.interpreter.intPool.put(mOff, cOff, l) return nil, nil } -func opExtCodeCopy(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opExtCodeCopy(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { var ( addr = common.BigToAddress(stack.pop()) mOff = stack.pop() cOff = stack.pop() l = stack.pop() ) - codeCopy := getData(env.StateDB.GetCode(addr), cOff, l) + codeCopy := getData(evm.StateDB.GetCode(addr), cOff, l) memory.Set(mOff.Uint64(), l.Uint64(), codeCopy) + + evm.interpreter.intPool.put(mOff, cOff, l) + return nil, nil } -func opGasprice(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(new(big.Int).Set(env.GasPrice)) +func opGasprice(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(evm.interpreter.intPool.get().Set(evm.GasPrice)) return nil, nil } -func opBlockhash(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opBlockhash(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { num := stack.pop() - n := new(big.Int).Sub(env.BlockNumber, common.Big257) - if num.Cmp(n) > 0 && num.Cmp(env.BlockNumber) < 0 { - stack.push(env.GetHash(num.Uint64()).Big()) + n := evm.interpreter.intPool.get().Sub(evm.BlockNumber, common.Big257) + if num.Cmp(n) > 0 && num.Cmp(evm.BlockNumber) < 0 { + stack.push(evm.GetHash(num.Uint64()).Big()) } else { stack.push(new(big.Int)) } + + evm.interpreter.intPool.put(num, n) return nil, nil } -func opCoinbase(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(env.Coinbase.Big()) +func opCoinbase(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(evm.Coinbase.Big()) return nil, nil } -func opTimestamp(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(U256(new(big.Int).Set(env.Time))) +func opTimestamp(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(U256(new(big.Int).Set(evm.Time))) return nil, nil } -func opNumber(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(U256(new(big.Int).Set(env.BlockNumber))) +func opNumber(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(U256(new(big.Int).Set(evm.BlockNumber))) return nil, nil } -func opDifficulty(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(U256(new(big.Int).Set(env.Difficulty))) +func opDifficulty(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(U256(new(big.Int).Set(evm.Difficulty))) return nil, nil } -func opGasLimit(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(U256(new(big.Int).Set(env.GasLimit))) +func opGasLimit(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(U256(new(big.Int).Set(evm.GasLimit))) return nil, nil } -func opPop(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.pop() +func opPop(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + evm.interpreter.intPool.put(stack.pop()) return nil, nil } -func opMload(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opMload(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { offset := stack.pop() val := common.BigD(memory.Get(offset.Int64(), 32)) stack.push(val) + + evm.interpreter.intPool.put(offset) return nil, nil } -func opMstore(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opMstore(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { // pop value of the stack mStart, val := stack.pop(), stack.pop() memory.Set(mStart.Uint64(), 32, common.BigToBytes(val, 256)) + + evm.interpreter.intPool.put(mStart, val) return nil, nil } -func opMstore8(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opMstore8(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { off, val := stack.pop().Int64(), stack.pop().Int64() memory.store[off] = byte(val & 0xff) + return nil, nil } -func opSload(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opSload(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { loc := common.BigToHash(stack.pop()) - val := env.StateDB.GetState(contract.Address(), loc).Big() + val := evm.StateDB.GetState(contract.Address(), loc).Big() stack.push(val) return nil, nil } -func opSstore(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opSstore(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { loc := common.BigToHash(stack.pop()) val := stack.pop() - env.StateDB.SetState(contract.Address(), loc, common.BigToHash(val)) + evm.StateDB.SetState(contract.Address(), loc, common.BigToHash(val)) + + evm.interpreter.intPool.put(val) return nil, nil } -func opJump(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opJump(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { pos := stack.pop() if !contract.jumpdests.has(contract.CodeHash, contract.Code, pos) { nop := contract.GetOp(pos.Uint64()) return nil, fmt.Errorf("invalid jump destination (%v) %v", nop, pos) } *pc = pos.Uint64() + + evm.interpreter.intPool.put(pos) return nil, nil } -func opJumpi(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opJumpi(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { pos, cond := stack.pop(), stack.pop() if cond.Cmp(common.BigTrue) >= 0 { if !contract.jumpdests.has(contract.CodeHash, contract.Code, pos) { @@ -445,57 +514,62 @@ func opJumpi(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *St } else { *pc++ } + + evm.interpreter.intPool.put(pos, cond) return nil, nil } -func opJumpdest(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opJumpdest(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { return nil, nil } -func opPc(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(new(big.Int).SetUint64(*pc)) +func opPc(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(evm.interpreter.intPool.get().SetUint64(*pc)) return nil, nil } -func opMsize(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(big.NewInt(int64(memory.Len()))) +func opMsize(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(evm.interpreter.intPool.get().SetInt64(int64(memory.Len()))) return nil, nil } -func opGas(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - stack.push(new(big.Int).Set(contract.Gas)) +func opGas(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + stack.push(evm.interpreter.intPool.get().SetUint64(contract.Gas)) return nil, nil } -func opCreate(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opCreate(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { var ( value = stack.pop() offset, size = stack.pop(), stack.pop() input = memory.Get(offset.Int64(), size.Int64()) - gas = new(big.Int).Set(contract.Gas) + gas = contract.Gas ) - if env.ChainConfig().IsEIP150(env.BlockNumber) { - gas.Div(gas, n64) - gas = gas.Sub(contract.Gas, gas) + if evm.ChainConfig().IsEIP150(evm.BlockNumber) { + gas -= gas / 64 } contract.UseGas(gas) - _, addr, suberr := env.Create(contract, input, gas, value) + _, addr, returnGas, suberr := evm.Create(contract, input, gas, value) // Push item on the stack based on the returned error. If the ruleset is // homestead we must check for CodeStoreOutOfGasError (homestead only // rule) and treat as an error, if the ruleset is frontier we must // ignore this error and pretend the operation was successful. - if env.ChainConfig().IsHomestead(env.BlockNumber) && suberr == ErrCodeStoreOutOfGas { + if evm.ChainConfig().IsHomestead(evm.BlockNumber) && suberr == ErrCodeStoreOutOfGas { stack.push(new(big.Int)) } else if suberr != nil && suberr != ErrCodeStoreOutOfGas { stack.push(new(big.Int)) } else { stack.push(addr.Big()) } + contract.Gas += returnGas + + evm.interpreter.intPool.put(value, offset, size) + return nil, nil } -func opCall(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - gas := stack.pop() +func opCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + gas := stack.pop().Uint64() // pop gas and value of the stack. addr, value := stack.pop(), stack.pop() value = U256(value) @@ -509,25 +583,26 @@ func opCall(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Sta // Get the arguments from the memory args := memory.Get(inOffset.Int64(), inSize.Int64()) - if len(value.Bytes()) > 0 { - gas.Add(gas, params.CallStipend) + if value.BitLen() > 0 { + gas += params.CallStipend } - ret, err := env.Call(contract, address, args, gas, value) - + ret, returnGas, err := evm.Call(contract, address, args, gas, value) if err != nil { stack.push(new(big.Int)) - } else { stack.push(big.NewInt(1)) memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } + contract.Gas += returnGas + + evm.interpreter.intPool.put(addr, value, inOffset, inSize, retOffset, retSize) return nil, nil } -func opCallCode(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - gas := stack.pop() +func opCallCode(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + gas := stack.pop().Uint64() // pop gas and value of the stack. addr, value := stack.pop(), stack.pop() value = U256(value) @@ -541,12 +616,11 @@ func opCallCode(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack // Get the arguments from the memory args := memory.Get(inOffset.Int64(), inSize.Int64()) - if len(value.Bytes()) > 0 { - gas.Add(gas, params.CallStipend) + if value.BitLen() > 0 { + gas += params.CallStipend } - ret, err := env.CallCode(contract, address, args, gas, value) - + ret, returnGas, err := evm.CallCode(contract, address, args, gas, value) if err != nil { stack.push(new(big.Int)) @@ -555,46 +629,54 @@ func opCallCode(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } + contract.Gas += returnGas + + evm.interpreter.intPool.put(addr, value, inOffset, inSize, retOffset, retSize) return nil, nil } -func opDelegateCall(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opDelegateCall(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { // if not homestead return an error. DELEGATECALL is not supported // during pre-homestead. - if !env.ChainConfig().IsHomestead(env.BlockNumber) { + if !evm.ChainConfig().IsHomestead(evm.BlockNumber) { return nil, fmt.Errorf("invalid opcode %x", DELEGATECALL) } - gas, to, inOffset, inSize, outOffset, outSize := stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() + gas, to, inOffset, inSize, outOffset, outSize := stack.pop().Uint64(), stack.pop(), stack.pop(), stack.pop(), stack.pop(), stack.pop() toAddr := common.BigToAddress(to) args := memory.Get(inOffset.Int64(), inSize.Int64()) - ret, err := env.DelegateCall(contract, toAddr, args, gas) + + ret, returnGas, err := evm.DelegateCall(contract, toAddr, args, gas) if err != nil { stack.push(new(big.Int)) } else { stack.push(big.NewInt(1)) memory.Set(outOffset.Uint64(), outSize.Uint64(), ret) } + contract.Gas += returnGas + + evm.interpreter.intPool.put(to, inOffset, inSize, outOffset, outSize) return nil, nil } -func opReturn(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opReturn(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { offset, size := stack.pop(), stack.pop() ret := memory.GetPtr(offset.Int64(), size.Int64()) + evm.interpreter.intPool.put(offset, size) return ret, nil } -func opStop(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { +func opStop(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { return nil, nil } -func opSuicide(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - balance := env.StateDB.GetBalance(contract.Address()) - env.StateDB.AddBalance(common.BigToAddress(stack.pop()), balance) +func opSuicide(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + balance := evm.StateDB.GetBalance(contract.Address()) + evm.StateDB.AddBalance(common.BigToAddress(stack.pop()), balance) - env.StateDB.Suicide(contract.Address()) + evm.StateDB.Suicide(contract.Address()) return nil, nil } @@ -603,7 +685,7 @@ func opSuicide(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack * // make log instruction function func makeLog(size int) executionFunc { - return func(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + return func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { topics := make([]common.Hash, size) mStart, mSize := stack.pop(), stack.pop() for i := 0; i < size; i++ { @@ -611,22 +693,24 @@ func makeLog(size int) executionFunc { } d := memory.Get(mStart.Int64(), mSize.Int64()) - env.StateDB.AddLog(&types.Log{ + evm.StateDB.AddLog(&types.Log{ Address: contract.Address(), Topics: topics, Data: d, // This is a non-consensus field, but assigned here because // core/state doesn't know the current block number. - BlockNumber: env.BlockNumber.Uint64(), + BlockNumber: evm.BlockNumber.Uint64(), }) + + evm.interpreter.intPool.put(mStart, mSize) return nil, nil } } // make push instruction function func makePush(size uint64, bsize *big.Int) executionFunc { - return func(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { - byts := getData(contract.Code, new(big.Int).SetUint64(*pc+1), bsize) + return func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + byts := getData(contract.Code, evm.interpreter.intPool.get().SetUint64(*pc+1), bsize) stack.push(common.Bytes2Big(byts)) *pc += size return nil, nil @@ -635,7 +719,7 @@ func makePush(size uint64, bsize *big.Int) executionFunc { // make push instruction function func makeDup(size int64) executionFunc { - return func(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + return func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { stack.dup(int(size)) return nil, nil } @@ -645,7 +729,7 @@ func makeDup(size int64) executionFunc { func makeSwap(size int64) executionFunc { // switch n + 1 otherwise n would be swapped with n size += 1 - return func(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + return func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { stack.swap(int(size)) return nil, nil } diff --git a/core/vm/int_pool_verifier.go b/core/vm/int_pool_verifier.go new file mode 100644 index 0000000000..61c83ba7e9 --- /dev/null +++ b/core/vm/int_pool_verifier.go @@ -0,0 +1,15 @@ +// +build VERIFY_EVM_INTEGER_POOL + +package vm + +import "fmt" + +const verifyPool = true + +func verifyIntegerPool(ip *intPool) { + for i, item := range ip.pool.data { + if item.Cmp(checkVal) != 0 { + panic(fmt.Sprintf("%d'th item failed aggressive pool check. Value was modified", i)) + } + } +} diff --git a/core/vm/int_pool_verifier_empty.go b/core/vm/int_pool_verifier_empty.go new file mode 100644 index 0000000000..982f8c6dd5 --- /dev/null +++ b/core/vm/int_pool_verifier_empty.go @@ -0,0 +1,7 @@ +// +build !VERIFY_EVM_INTEGER_POOL + +package vm + +const verifyPool = false + +func verifyIntegerPool(ip *intPool) {} diff --git a/core/vm/vm.go b/core/vm/interpreter.go similarity index 78% rename from core/vm/vm.go rename to core/vm/interpreter.go index a5f48750d1..ad41e36022 100644 --- a/core/vm/vm.go +++ b/core/vm/interpreter.go @@ -23,6 +23,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -60,6 +61,7 @@ type Interpreter struct { env *EVM cfg Config gasTable params.GasTable + intPool *intPool } // NewInterpreter returns a new instance of the Interpreter. @@ -75,6 +77,7 @@ func NewInterpreter(env *EVM, cfg Config) *Interpreter { env: env, cfg: cfg, gasTable: env.ChainConfig().GasTable(env.BlockNumber), + intPool: newIntPool(), } } @@ -106,14 +109,18 @@ func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err e // For optimisation reason we're using uint64 as the program counter. // It's theoretically possible to go above 2^64. The YP defines the PC to be uint256. Practically much less so feasible. pc = uint64(0) // program counter - cost *big.Int + cost uint64 ) contract.Input = input // User defer pattern to check for an error and, based on the error being nil or not, use all gas and return. defer func() { if err != nil && evm.cfg.Debug { - evm.cfg.Tracer.CaptureState(evm.env, pc, op, contract.Gas, cost, mem, stack, contract, evm.env.depth, err) + // XXX For debugging + //fmt.Printf("%04d: %8v cost = %-8d stack = %-8d ERR = %v\n", pc, op, cost, stack.len(), err) + // TODO update the tracer + g, c := new(big.Int).SetUint64(contract.Gas), new(big.Int).SetUint64(cost) + evm.cfg.Tracer.CaptureState(evm.env, pc, op, g, c, mem, stack, contract, evm.env.depth, err) } }() @@ -126,7 +133,7 @@ func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err e } // The Interpreter main run loop (contextual). This loop runs until either an - // explicit STOP, RETURN or SUICIDE is executed, an error accured during + // explicit STOP, RETURN or SUICIDE is executed, an error occurred during // the execution of one of the operations or until the evm.done is set by // the parent context.Context. for atomic.LoadInt32(&evm.env.abort) == 0 { @@ -147,34 +154,47 @@ func (evm *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err e return nil, err } - var memorySize *big.Int + var memorySize uint64 // calculate the new memory size and expand the memory to fit // the operation if operation.memorySize != nil { - memorySize = operation.memorySize(stack) + memSize, overflow := bigUint64(operation.memorySize(stack)) + if overflow { + return nil, errGasUintOverflow + } // memory is expanded in words of 32 bytes. Gas // is also calculated in words. - memorySize.Mul(toWordSize(memorySize), big.NewInt(32)) + if memorySize, overflow = math.SafeMul(toWordSize(memSize), 32); overflow { + return nil, errGasUintOverflow + } } if !evm.cfg.DisableGasMetering { // consume the gas and return an error if not enough gas is available. // cost is explicitly set so that the capture state defer method cas get the proper cost - cost = operation.gasCost(evm.gasTable, evm.env, contract, stack, mem, memorySize) - if !contract.UseGas(cost) { + cost, err = operation.gasCost(evm.gasTable, evm.env, contract, stack, mem, memorySize) + if err != nil || !contract.UseGas(cost) { return nil, ErrOutOfGas } } - if memorySize != nil { - mem.Resize(memorySize.Uint64()) + if memorySize > 0 { + mem.Resize(memorySize) } if evm.cfg.Debug { - evm.cfg.Tracer.CaptureState(evm.env, pc, op, contract.Gas, cost, mem, stack, contract, evm.env.depth, err) + g, c := new(big.Int).SetUint64(contract.Gas), new(big.Int).SetUint64(cost) + evm.cfg.Tracer.CaptureState(evm.env, pc, op, g, c, mem, stack, contract, evm.env.depth, err) } + // XXX For debugging + //fmt.Printf("%04d: %8v cost = %-8d stack = %-8d\n", pc, op, cost, stack.len()) // execute the operation res, err := operation.execute(&pc, evm.env, contract, mem, stack) + // verifyPool is a build flag. Pool verification makes sure the integrity + // of the integer pool by comparing values to a default value. + if verifyPool { + verifyIntegerPool(evm.intPool) + } switch { case err != nil: return nil, err diff --git a/core/vm/virtual_machine.go b/core/vm/intpool.go similarity index 53% rename from core/vm/virtual_machine.go rename to core/vm/intpool.go index 6291088846..4f1228e149 100644 --- a/core/vm/virtual_machine.go +++ b/core/vm/intpool.go @@ -1,4 +1,4 @@ -// Copyright 2014 The go-ethereum Authors +// Copyright 2017 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -16,7 +16,34 @@ package vm -// VirtualMachine is an EVM interface -type VirtualMachine interface { - Run(*Contract, []byte) ([]byte, error) +import "math/big" + +var checkVal = big.NewInt(-42) + +// intPool is a pool of big integers that +// can be reused for all big.Int operations. +type intPool struct { + pool *Stack +} + +func newIntPool() *intPool { + return &intPool{pool: newstack()} +} + +func (p *intPool) get() *big.Int { + if p.pool.len() > 0 { + return p.pool.pop() + } + return new(big.Int) +} +func (p *intPool) put(is ...*big.Int) { + for _, i := range is { + // verifyPool is a build flag. Pool verification makes sure the integrity + // of the integer pool by comparing values to a default value. + if verifyPool { + i.Set(checkVal) + } + + p.pool.push(i) + } } diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index f4ce818833..80e12c10b5 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -17,6 +17,7 @@ package vm import ( + "errors" "math/big" "github.com/ethereum/go-ethereum/params" @@ -24,11 +25,13 @@ import ( type ( executionFunc func(pc *uint64, env *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) - gasFunc func(params.GasTable, *EVM, *Contract, *Stack, *Memory, *big.Int) *big.Int + gasFunc func(params.GasTable, *EVM, *Contract, *Stack, *Memory, uint64) (uint64, error) // last parameter is the requested memory size as a uint64 stackValidationFunc func(*Stack) error memorySizeFunc func(*Stack) *big.Int ) +var errGasUintOverflow = errors.New("gas uint64 overflow") + type operation struct { // op is the operation function execute executionFunc @@ -431,7 +434,7 @@ func NewJumpTable() [256]operation { }, STOP: { execute: opStop, - gasCost: constGasFunc(Zero), + gasCost: constGasFunc(0), validateStack: makeStackFunc(0, 0), halts: true, valid: true, diff --git a/core/vm/logger_test.go b/core/vm/logger_test.go index 1d0bd96fad..ca60cba43e 100644 --- a/core/vm/logger_test.go +++ b/core/vm/logger_test.go @@ -56,7 +56,7 @@ func TestStoreCapture(t *testing.T) { logger = NewStructLogger(nil) mem = NewMemory() stack = newstack() - contract = NewContract(&dummyContractRef{}, &dummyContractRef{}, new(big.Int), new(big.Int)) + contract = NewContract(&dummyContractRef{}, &dummyContractRef{}, new(big.Int), 0) ) stack.push(big.NewInt(1)) stack.push(big.NewInt(0)) @@ -78,7 +78,7 @@ func TestStorageCapture(t *testing.T) { t.Skip("implementing this function is difficult. it requires all sort of interfaces to be implemented which isn't trivial. The value (the actual test) isn't worth it") var ( ref = &dummyContractRef{} - contract = NewContract(ref, ref, new(big.Int), new(big.Int)) + contract = NewContract(ref, ref, new(big.Int), 0) env = NewEVM(Context{}, dummyStateDB{ref: ref}, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) logger = NewStructLogger(nil) mem = NewMemory() diff --git a/core/vm/memory.go b/core/vm/memory.go index d011884173..99a84d2271 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -20,11 +20,12 @@ import "fmt" // Memory implements a simple memory model for the ethereum virtual machine. type Memory struct { - store []byte + store []byte + lastGasCost uint64 } func NewMemory() *Memory { - return &Memory{nil} + return &Memory{} } // Set sets offset + size to value diff --git a/core/vm/runtime/env.go b/core/vm/runtime/env.go index a25c6d71c6..9aa88e6690 100644 --- a/core/vm/runtime/env.go +++ b/core/vm/runtime/env.go @@ -36,7 +36,7 @@ func NewEnv(cfg *Config, state *state.StateDB) *vm.EVM { BlockNumber: cfg.BlockNumber, Time: cfg.Time, Difficulty: cfg.Difficulty, - GasLimit: cfg.GasLimit, + GasLimit: new(big.Int).SetUint64(cfg.GasLimit), GasPrice: new(big.Int), } diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index b5adb982c3..cf46603dbd 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -17,6 +17,7 @@ package runtime import ( + "math" "math/big" "time" @@ -37,7 +38,7 @@ type Config struct { Coinbase common.Address BlockNumber *big.Int Time *big.Int - GasLimit *big.Int + GasLimit uint64 GasPrice *big.Int Value *big.Int DisableJit bool // "disable" so it's enabled by default @@ -68,8 +69,8 @@ func setDefaults(cfg *Config) { if cfg.Time == nil { cfg.Time = big.NewInt(time.Now().Unix()) } - if cfg.GasLimit == nil { - cfg.GasLimit = new(big.Int).Set(common.MaxBig) + if cfg.GasLimit == 0 { + cfg.GasLimit = math.MaxUint64 } if cfg.GasPrice == nil { cfg.GasPrice = new(big.Int) @@ -112,7 +113,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { receiver.SetCode(crypto.Keccak256Hash(code), code) // Call the code with the given configuration. - ret, err := vmenv.Call( + ret, _, err := vmenv.Call( sender, receiver.Address(), input, @@ -140,12 +141,13 @@ func Create(input []byte, cfg *Config) ([]byte, common.Address, error) { ) // Call the code with the given configuration. - return vmenv.Create( + code, address, _, err := vmenv.Create( sender, input, cfg.GasLimit, cfg.Value, ) + return code, address, err } // Call executes the code given by the contract's address. It will return the @@ -160,7 +162,7 @@ func Call(address common.Address, input []byte, cfg *Config) ([]byte, error) { sender := cfg.State.GetOrNewStateObject(cfg.Origin) // Call the code with the given configuration. - ret, err := vmenv.Call( + ret, _, err := vmenv.Call( sender, address, input, diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 1e618b688a..8ad74a89ab 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -39,8 +39,8 @@ func TestDefaults(t *testing.T) { if cfg.Time == nil { t.Error("expected time to be non nil") } - if cfg.GasLimit == nil { - t.Error("expected time to be non nil") + if cfg.GasLimit == 0 { + t.Error("didn't expect gaslimit to be zero") } if cfg.GasPrice == nil { t.Error("expected time to be non nil") diff --git a/core/vm/stack_table.go b/core/vm/stack_table.go index ce4727a714..0936ef06fc 100644 --- a/core/vm/stack_table.go +++ b/core/vm/stack_table.go @@ -12,8 +12,8 @@ func makeStackFunc(pop, push int) stackValidationFunc { return err } - if push > 0 && int64(stack.len()-pop+push) > params.StackLimit.Int64() { - return fmt.Errorf("stack limit reached %d (%d)", stack.len(), params.StackLimit.Int64()) + if push > 0 && stack.len()-pop+push > int(params.StackLimit) { + return fmt.Errorf("stack limit reached %d (%d)", stack.len(), params.StackLimit) } return nil } diff --git a/eth/api_backend.go b/eth/api_backend.go index 1174588ea7..72ed76cc4a 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -106,14 +106,14 @@ func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *EthApiBackend) GetVMEnv(ctx context.Context, msg core.Message, state ethapi.State, header *types.Header) (*vm.EVM, func() error, error) { +func (b *EthApiBackend) GetEVM(ctx context.Context, msg core.Message, state ethapi.State, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { statedb := state.(EthApiState).state from := statedb.GetOrNewStateObject(msg.From()) from.SetBalance(common.MaxBig) vmError := func() error { return nil } context := core.NewEVMContext(msg, header, b.eth.BlockChain()) - return vm.NewEVM(context, statedb, b.eth.chainConfig, vm.Config{}), vmError, nil + return vm.NewEVM(context, statedb, b.eth.chainConfig, vmCfg), vmError, nil } func (b *EthApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { diff --git a/eth/bind.go b/eth/bind.go index a9864b3677..2ee9f2bf7b 100644 --- a/eth/bind.go +++ b/eth/bind.go @@ -69,7 +69,7 @@ func (b *ContractBackend) PendingCodeAt(ctx context.Context, contract common.Add // against the pending block, not the stable head of the chain. func (b *ContractBackend) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNum *big.Int) ([]byte, error) { out, err := b.bcapi.Call(ctx, toCallArgs(msg), toBlockNumber(blockNum)) - return common.FromHex(out), err + return out, err } // ContractCall implements bind.ContractCaller executing an Ethereum contract @@ -77,7 +77,7 @@ func (b *ContractBackend) CallContract(ctx context.Context, msg ethereum.CallMsg // against the pending block, not the stable head of the chain. func (b *ContractBackend) PendingCallContract(ctx context.Context, msg ethereum.CallMsg) ([]byte, error) { out, err := b.bcapi.Call(ctx, toCallArgs(msg), rpc.PendingBlockNumber) - return common.FromHex(out), err + return out, err } func toCallArgs(msg ethereum.CallMsg) ethapi.CallArgs { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 9be4bd87d0..7e2952439a 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -49,12 +49,12 @@ var ( MaxReceiptFetch = 256 // Amount of transaction receipts to allow fetching per request MaxStateFetch = 384 // Amount of node state values to allow fetching per request - MaxForkAncestry = 3 * params.EpochDuration.Uint64() // Maximum chain reorganisation - rttMinEstimate = 2 * time.Second // Minimum round-trip time to target for download requests - rttMaxEstimate = 20 * time.Second // Maximum rount-trip time to target for download requests - rttMinConfidence = 0.1 // Worse confidence factor in our estimated RTT value - ttlScaling = 3 // Constant scaling factor for RTT -> TTL conversion - ttlLimit = time.Minute // Maximum TTL allowance to prevent reaching crazy timeouts + MaxForkAncestry = 3 * params.EpochDuration // Maximum chain reorganisation + rttMinEstimate = 2 * time.Second // Minimum round-trip time to target for download requests + rttMaxEstimate = 20 * time.Second // Maximum rount-trip time to target for download requests + rttMinConfidence = 0.1 // Worse confidence factor in our estimated RTT value + ttlScaling = 3 // Constant scaling factor for RTT -> TTL conversion + ttlLimit = time.Minute // Maximum TTL allowance to prevent reaching crazy timeouts qosTuningPeers = 5 // Number of peers to tune based on (best peers) qosConfidenceCap = 10 // Number of peers above which not to modify RTT confidence diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index b156c471df..a9ea797eac 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -119,7 +119,7 @@ func (dl *downloadTester) makeChain(n int, seed byte, parent *types.Block, paren // If the block number is multiple of 3, send a bonus transaction to the miner if parent == dl.genesis && i%3 == 0 { signer := types.MakeSigner(params.TestChainConfig, block.Number()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), new(big.Int).SetUint64(params.TxGas), nil, nil), signer, testKey) if err != nil { panic(err) } diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/fetcher_test.go index 2e28541ab3..7a94241c66 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/fetcher_test.go @@ -51,7 +51,7 @@ func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common // If the block number is multiple of 3, send a bonus transaction to the miner if parent == genesis && i%3 == 0 { signer := types.MakeSigner(params.TestChainConfig, block.Number()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil), signer, testKey) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), new(big.Int).SetUint64(params.TxGas), nil, nil), signer, testKey) if err != nil { panic(err) } diff --git a/eth/handler_test.go b/eth/handler_test.go index 8a5d7173b3..03d5404a4d 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -36,6 +36,8 @@ import ( "github.com/ethereum/go-ethereum/params" ) +var bigTxGas = new(big.Int).SetUint64(params.TxGas) + // Tests that protocol versions and modes of operations are matched up properly. func TestProtocolCompatibility(t *testing.T) { // Define the compatibility chart @@ -312,13 +314,13 @@ func testGetNodeData(t *testing.T, protocol int) { switch i { case 0: // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank.Address), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank.Address), acc1Addr, big.NewInt(10000), bigTxGas, nil, nil), signer, testBankKey) block.AddTx(tx) case 1: // In block 2, the test bank sends some more ether to account #1. // acc1Addr passes it on to account #2. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank.Address), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) - tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank.Address), acc1Addr, big.NewInt(1000), bigTxGas, nil, nil), signer, testBankKey) + tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1000), bigTxGas, nil, nil), signer, acc1Key) block.AddTx(tx1) block.AddTx(tx2) case 2: @@ -404,13 +406,13 @@ func testGetReceipt(t *testing.T, protocol int) { switch i { case 0: // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank.Address), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank.Address), acc1Addr, big.NewInt(10000), bigTxGas, nil, nil), signer, testBankKey) block.AddTx(tx) case 1: // In block 2, the test bank sends some more ether to account #1. // acc1Addr passes it on to account #2. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank.Address), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) - tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBank.Address), acc1Addr, big.NewInt(1000), bigTxGas, nil, nil), signer, testBankKey) + tx2, _ := types.SignTx(types.NewTransaction(block.TxNonce(acc1Addr), acc2Addr, big.NewInt(1000), bigTxGas, nil, nil), signer, acc1Key) block.AddTx(tx1) block.AddTx(tx2) case 2: diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 925f547b60..4c8e784c5d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -47,6 +47,8 @@ import ( const defaultGas = 90000 +var emptyHex = "0x" + // PublicEthereumAPI provides an API to access Ethereum related information. // It offers only methods that operate on public data that is freely available to anyone. type PublicEthereumAPI struct { @@ -503,58 +505,76 @@ type CallArgs struct { Data hexutil.Bytes `json:"data"` } -func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber) (string, *big.Int, error) { +func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber, vmCfg vm.Config) ([]byte, *big.Int, error) { defer func(start time.Time) { glog.V(logger.Debug).Infof("call took %v", time.Since(start)) }(time.Now()) state, header, err := s.b.StateAndHeaderByNumber(ctx, blockNr) if state == nil || err != nil { - return "0x", common.Big0, err + return nil, common.Big0, err } - // Set the account address to interact with - var addr common.Address - if args.From == (common.Address{}) { + // Set sender address or use a default if none specified + addr := args.From + if addr == (common.Address{}) { accounts := s.b.AccountManager().Accounts() - if len(accounts) == 0 { - addr = common.Address{} - } else { + if len(accounts) > 0 { addr = accounts[0].Address } - } else { - addr = args.From } - // Assemble the CALL invocation + // Set default gas & gas price if none were set gas, gasPrice := args.Gas.ToInt(), args.GasPrice.ToInt() - if gas.Cmp(common.Big0) == 0 { + if gas.BitLen() == 0 { gas = big.NewInt(50000000) } - if gasPrice.Cmp(common.Big0) == 0 { + if gasPrice.BitLen() == 0 { gasPrice = new(big.Int).Mul(big.NewInt(50), common.Shannon) } + + // Create new call message msg := types.NewMessage(addr, args.To, 0, args.Value.ToInt(), gas, gasPrice, args.Data, false) - // Execute the call and return - vmenv, vmError, err := s.b.GetVMEnv(ctx, msg, state, header) + // Setup context so it may be cancelled the call has completed + // or, in case of unmetered gas, setup a context with a timeout. + var cancel context.CancelFunc + if vmCfg.DisableGasMetering { + ctx, cancel = context.WithTimeout(ctx, time.Second*5) + } else { + ctx, cancel = context.WithCancel(ctx) + } + // Make sure the context is cancelled when the call has completed + // this makes sure resources are cleaned up. + defer func() { cancel() }() + + // Get a new instance of the EVM. + evm, vmError, err := s.b.GetEVM(ctx, msg, state, header, vmCfg) if err != nil { - return "0x", common.Big0, err + return nil, common.Big0, err } + // Wait for the context to be done and cancel the evm. Even if the + // EVM has finished, cancelling may be done (repeatedly) + go func() { + select { + case <-ctx.Done(): + evm.Cancel() + } + }() + + // Setup the gas pool (also for unmetered requests) + // and apply the message. gp := new(core.GasPool).AddGas(common.MaxBig) - res, gas, err := core.ApplyMessage(vmenv, msg, gp) + res, gas, err := core.ApplyMessage(evm, msg, gp) if err := vmError(); err != nil { - return "0x", common.Big0, err + return nil, common.Big0, err } - if len(res) == 0 { // backwards compatibility - return "0x", gas, err - } - return common.ToHex(res), gas, err + return res, gas, err } // Call executes the given transaction on the state for the given block number. // It doesn't make and changes in the state/blockchain and is useful to execute and retrieve values. -func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber) (string, error) { - result, _, err := s.doCall(ctx, args, blockNr) - return result, err +func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { + result, _, err := s.doCall(ctx, args, blockNr, vm.Config{DisableGasMetering: true}) + return (hexutil.Bytes)(result), err } // EstimateGas returns an estimate of the amount of gas needed to execute the given transaction. @@ -576,7 +596,7 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (* mid := (hi + lo) / 2 (*big.Int)(&args.Gas).SetUint64(mid) - _, gas, err := s.doCall(ctx, args, rpc.PendingBlockNumber) + _, gas, err := s.doCall(ctx, args, rpc.PendingBlockNumber, vm.Config{}) // If the transaction became invalid or used all the gas (failed), raise the gas limit if err != nil || gas.Cmp((*big.Int)(&args.Gas)) == 0 { diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index ebb14a5b5c..214214f511 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -51,7 +51,7 @@ type Backend interface { GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetTd(blockHash common.Hash) *big.Int - GetVMEnv(ctx context.Context, msg core.Message, state State, header *types.Header) (*vm.EVM, func() error, error) + GetEVM(ctx context.Context, msg core.Message, state State, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) // TxPool API SendTx(ctx context.Context, signedTx *types.Transaction) error RemoveTx(txHash common.Hash) diff --git a/internal/ethapi/tracer_test.go b/internal/ethapi/tracer_test.go index 65a23f55e7..693afe802f 100644 --- a/internal/ethapi/tracer_test.go +++ b/internal/ethapi/tracer_test.go @@ -45,7 +45,7 @@ func (account) ForEachStorage(cb func(key, value common.Hash) bool) {} func runTrace(tracer *JavascriptTracer) (interface{}, error) { env := vm.NewEVM(vm.Context{}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) - contract := vm.NewContract(account{}, account{}, big.NewInt(0), big.NewInt(10000)) + contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} _, err := env.Interpreter().Run(contract, []byte{}) @@ -134,7 +134,7 @@ func TestHaltBetweenSteps(t *testing.T) { } env := vm.NewEVM(vm.Context{}, nil, params.TestChainConfig, vm.Config{Debug: true, Tracer: tracer}) - contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), big.NewInt(0)) + contract := vm.NewContract(&account{}, &account{}, big.NewInt(0), 0) tracer.CaptureState(env, 0, 0, big.NewInt(0), big.NewInt(0), nil, nil, contract, 0, nil) timeout := errors.New("stahp") diff --git a/les/api_backend.go b/les/api_backend.go index 3a71ac4e02..ed2a7cd13e 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -88,7 +88,7 @@ func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *LesApiBackend) GetVMEnv(ctx context.Context, msg core.Message, state ethapi.State, header *types.Header) (*vm.EVM, func() error, error) { +func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state ethapi.State, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { stateDb := state.(*light.LightState).Copy() addr := msg.From() from, err := stateDb.GetOrNewStateObject(ctx, addr) @@ -99,7 +99,7 @@ func (b *LesApiBackend) GetVMEnv(ctx context.Context, msg core.Message, state et vmstate := light.NewVMState(ctx, stateDb) context := core.NewEVMContext(msg, header, b.eth.blockchain) - return vm.NewEVM(context, vmstate, b.eth.chainConfig, vm.Config{}), vmstate.Error, nil + return vm.NewEVM(context, vmstate, b.eth.chainConfig, vmCfg), vmstate.Error, nil } func (b *LesApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { diff --git a/les/helper_test.go b/les/helper_test.go index e0b7558eeb..2c6f34a92e 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -57,6 +57,8 @@ var ( testContractDeployed = uint64(2) testBufLimit = uint64(100) + + bigTxGas = new(big.Int).SetUint64(params.TxGas) ) /* @@ -80,15 +82,15 @@ func testChainGen(i int, block *core.BlockGen) { switch i { case 0: // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(10000), bigTxGas, nil, nil), signer, testBankKey) block.AddTx(tx) case 1: // In block 2, the test bank sends some more ether to account #1. // acc1Addr passes it on to account #2. // acc1Addr creates a test contract. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), bigTxGas, nil, nil), signer, testBankKey) nonce := block.TxNonce(acc1Addr) - tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) + tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), bigTxGas, nil, nil), signer, acc1Key) nonce++ tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), big.NewInt(200000), big.NewInt(0), testContractCode), signer, acc1Key) testContractAddr = crypto.CreateAddress(acc1Addr, nonce) diff --git a/light/odr_test.go b/light/odr_test.go index a2f969acb3..a76050a299 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -49,6 +49,8 @@ var ( testContractCode = common.Hex2Bytes("606060405260cc8060106000396000f360606040526000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146041578063c16431b914606b57603f565b005b6055600480803590602001909190505060a9565b6040518082815260200191505060405180910390f35b60886004808035906020019091908035906020019091905050608a565b005b80600060005083606481101560025790900160005b50819055505b5050565b6000600060005082606481101560025790900160005b5054905060c7565b91905056") testContractAddr common.Address + + bigTxGas = new(big.Int).SetUint64(params.TxGas) ) type testOdr struct { @@ -205,15 +207,15 @@ func testChainGen(i int, block *core.BlockGen) { switch i { case 0: // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, testBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(10000), bigTxGas, nil, nil), signer, testBankKey) block.AddTx(tx) case 1: // In block 2, the test bank sends some more ether to account #1. // acc1Addr passes it on to account #2. // acc1Addr creates a test contract. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, testBankKey) + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1000), bigTxGas, nil, nil), signer, testBankKey) nonce := block.TxNonce(acc1Addr) - tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, acc1Key) + tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1000), bigTxGas, nil, nil), signer, acc1Key) nonce++ tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), big.NewInt(1000000), big.NewInt(0), testContractCode), signer, acc1Key) testContractAddr = crypto.CreateAddress(acc1Addr, nonce) diff --git a/light/txpool_test.go b/light/txpool_test.go index d8f04e6173..af2dcbbef1 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -77,7 +77,7 @@ func txPoolTestChainGen(i int, block *core.BlockGen) { func TestTxPool(t *testing.T) { for i := range testTx { - testTx[i], _ = types.SignTx(types.NewTransaction(uint64(i), acc1Addr, big.NewInt(10000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + testTx[i], _ = types.SignTx(types.NewTransaction(uint64(i), acc1Addr, big.NewInt(10000), bigTxGas, nil, nil), types.HomesteadSigner{}, testBankKey) } var ( diff --git a/miner/miner.go b/miner/miner.go index 61cd3e0499..83059f4b1d 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -171,7 +171,7 @@ func (self *Miner) HashRate() (tot int64) { } func (self *Miner) SetExtra(extra []byte) error { - if uint64(len(extra)) > params.MaximumExtraDataSize.Uint64() { + if uint64(len(extra)) > params.MaximumExtraDataSize { return fmt.Errorf("Extra exceeds max length. %d > %v", len(extra), params.MaximumExtraDataSize) } self.worker.setExtra(extra) diff --git a/params/gas_table.go b/params/gas_table.go index 093dacc8b3..a060539041 100644 --- a/params/gas_table.go +++ b/params/gas_table.go @@ -16,41 +16,35 @@ package params -import "math/big" - type GasTable struct { - ExtcodeSize *big.Int - ExtcodeCopy *big.Int - Balance *big.Int - SLoad *big.Int - Calls *big.Int - Suicide *big.Int + ExtcodeSize uint64 + ExtcodeCopy uint64 + Balance uint64 + SLoad uint64 + Calls uint64 + Suicide uint64 - ExpByte *big.Int + ExpByte uint64 // CreateBySuicide occurs when the // refunded account is one that does // not exist. This logic is similar // to call. May be left nil. Nil means // not charged. - CreateBySuicide *big.Int + CreateBySuicide uint64 } var ( // GasTableHomestead contain the gas prices for // the homestead phase. GasTableHomestead = GasTable{ - ExtcodeSize: big.NewInt(20), - ExtcodeCopy: big.NewInt(20), - Balance: big.NewInt(20), - SLoad: big.NewInt(50), - Calls: big.NewInt(40), - Suicide: big.NewInt(0), - ExpByte: big.NewInt(10), - - // explicitly set to nil to indicate - // this rule does not apply to homestead. - CreateBySuicide: nil, + ExtcodeSize: 20, + ExtcodeCopy: 20, + Balance: 20, + SLoad: 50, + Calls: 40, + Suicide: 0, + ExpByte: 10, } // GasTableHomestead contain the gas re-prices for @@ -58,26 +52,26 @@ var ( // // TODO rename to GasTableEIP150 GasTableHomesteadGasRepriceFork = GasTable{ - ExtcodeSize: big.NewInt(700), - ExtcodeCopy: big.NewInt(700), - Balance: big.NewInt(400), - SLoad: big.NewInt(200), - Calls: big.NewInt(700), - Suicide: big.NewInt(5000), - ExpByte: big.NewInt(10), + ExtcodeSize: 700, + ExtcodeCopy: 700, + Balance: 400, + SLoad: 200, + Calls: 700, + Suicide: 5000, + ExpByte: 10, - CreateBySuicide: big.NewInt(25000), + CreateBySuicide: 25000, } GasTableEIP158 = GasTable{ - ExtcodeSize: big.NewInt(700), - ExtcodeCopy: big.NewInt(700), - Balance: big.NewInt(400), - SLoad: big.NewInt(200), - Calls: big.NewInt(700), - Suicide: big.NewInt(5000), - ExpByte: big.NewInt(50), + ExtcodeSize: 700, + ExtcodeCopy: 700, + Balance: 400, + SLoad: 200, + Calls: 700, + Suicide: 5000, + ExpByte: 50, - CreateBySuicide: big.NewInt(25000), + CreateBySuicide: 25000, } ) diff --git a/params/protocol_params.go b/params/protocol_params.go index f5b6bedeb7..f48bf4992d 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -18,56 +18,58 @@ package params import "math/big" -var ( - MaximumExtraDataSize = big.NewInt(32) // Maximum size extra data may be after Genesis. - ExpByteGas = big.NewInt(10) // Times ceil(log256(exponent)) for the EXP instruction. - SloadGas = big.NewInt(50) // Multiplied by the number of 32-byte words that are copied (round up) for any *COPY operation and added. - CallValueTransferGas = big.NewInt(9000) // Paid for CALL when the value transfer is non-zero. - CallNewAccountGas = big.NewInt(25000) // Paid for CALL when the destination address didn't exist prior. - TxGas = big.NewInt(21000) // Per transaction not creating a contract. NOTE: Not payable on data of calls between transactions. - TxGasContractCreation = big.NewInt(53000) // Per transaction that creates a contract. NOTE: Not payable on data of calls between transactions. - TxDataZeroGas = big.NewInt(4) // Per byte of data attached to a transaction that equals zero. NOTE: Not payable on data of calls between transactions. - DifficultyBoundDivisor = big.NewInt(2048) // The bound divisor of the difficulty, used in the update calculations. - QuadCoeffDiv = big.NewInt(512) // Divisor for the quadratic particle of the memory cost equation. - GenesisDifficulty = big.NewInt(131072) // Difficulty of the Genesis block. - DurationLimit = big.NewInt(13) // The decision boundary on the blocktime duration used to determine whether difficulty should go up or not. - SstoreSetGas = big.NewInt(20000) // Once per SLOAD operation. - LogDataGas = big.NewInt(8) // Per byte in a LOG* operation's data. - CallStipend = big.NewInt(2300) // Free gas given at beginning of call. - EcrecoverGas = big.NewInt(3000) // - Sha256WordGas = big.NewInt(12) // +const ( + MaximumExtraDataSize uint64 = 32 // Maximum size extra data may be after Genesis. + ExpByteGas uint64 = 10 // Times ceil(log256(exponent)) for the EXP instruction. + SloadGas uint64 = 50 // Multiplied by the number of 32-byte words that are copied (round up) for any *COPY operation and added. + CallValueTransferGas uint64 = 9000 // Paid for CALL when the value transfer is non-zero. + CallNewAccountGas uint64 = 25000 // Paid for CALL when the destination address didn't exist prior. + TxGas uint64 = 21000 // Per transaction not creating a contract. NOTE: Not payable on data of calls between transactions. + TxGasContractCreation uint64 = 53000 // Per transaction that creates a contract. NOTE: Not payable on data of calls between transactions. + TxDataZeroGas uint64 = 4 // Per byte of data attached to a transaction that equals zero. NOTE: Not payable on data of calls between transactions. + QuadCoeffDiv uint64 = 512 // Divisor for the quadratic particle of the memory cost equation. + SstoreSetGas uint64 = 20000 // Once per SLOAD operation. + LogDataGas uint64 = 8 // Per byte in a LOG* operation's data. + CallStipend uint64 = 2300 // Free gas given at beginning of call. + EcrecoverGas uint64 = 3000 // + Sha256WordGas uint64 = 12 // - MinGasLimit = big.NewInt(5000) // Minimum the gas limit may ever be. - GenesisGasLimit = big.NewInt(4712388) // Gas limit of the Genesis block. - TargetGasLimit = new(big.Int).Set(GenesisGasLimit) // The artificial target - - Sha3Gas = big.NewInt(30) // Once per SHA3 operation. - Sha256Gas = big.NewInt(60) // - IdentityWordGas = big.NewInt(3) // - Sha3WordGas = big.NewInt(6) // Once per word of the SHA3 operation's data. - SstoreResetGas = big.NewInt(5000) // Once per SSTORE operation if the zeroness changes from zero. - SstoreClearGas = big.NewInt(5000) // Once per SSTORE operation if the zeroness doesn't change. - SstoreRefundGas = big.NewInt(15000) // Once per SSTORE operation if the zeroness changes to zero. - JumpdestGas = big.NewInt(1) // Refunded gas, once per SSTORE operation if the zeroness changes to zero. - IdentityGas = big.NewInt(15) // - GasLimitBoundDivisor = big.NewInt(1024) // The bound divisor of the gas limit, used in update calculations. - EpochDuration = big.NewInt(30000) // Duration between proof-of-work epochs. - CallGas = big.NewInt(40) // Once per CALL operation & message call transaction. - CreateDataGas = big.NewInt(200) // - Ripemd160Gas = big.NewInt(600) // - Ripemd160WordGas = big.NewInt(120) // - MinimumDifficulty = big.NewInt(131072) // The minimum that the difficulty may ever be. - CallCreateDepth = big.NewInt(1024) // Maximum depth of call/create stack. - ExpGas = big.NewInt(10) // Once per EXP instruction. - LogGas = big.NewInt(375) // Per LOG* operation. - CopyGas = big.NewInt(3) // - StackLimit = big.NewInt(1024) // Maximum size of VM stack allowed. - TierStepGas = big.NewInt(0) // Once per operation, for a selection of them. - LogTopicGas = big.NewInt(375) // Multiplied by the * of the LOG*, per LOG transaction. e.g. LOG0 incurs 0 * c_txLogTopicGas, LOG4 incurs 4 * c_txLogTopicGas. - CreateGas = big.NewInt(32000) // Once per CREATE operation & contract-creation transaction. - SuicideRefundGas = big.NewInt(24000) // Refunded following a suicide operation. - MemoryGas = big.NewInt(3) // Times the address of the (highest referenced byte in memory + 1). NOTE: referencing happens on read, write and in instructions such as RETURN and CALL. - TxDataNonZeroGas = big.NewInt(68) // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. + Sha3Gas uint64 = 30 // Once per SHA3 operation. + Sha256Gas uint64 = 60 // + IdentityWordGas uint64 = 3 // + Sha3WordGas uint64 = 6 // Once per word of the SHA3 operation's data. + SstoreResetGas uint64 = 5000 // Once per SSTORE operation if the zeroness changes from zero. + SstoreClearGas uint64 = 5000 // Once per SSTORE operation if the zeroness doesn't change. + SstoreRefundGas uint64 = 15000 // Once per SSTORE operation if the zeroness changes to zero. + JumpdestGas uint64 = 1 // Refunded gas, once per SSTORE operation if the zeroness changes to zero. + IdentityGas uint64 = 15 // + EpochDuration uint64 = 30000 // Duration between proof-of-work epochs. + CallGas uint64 = 40 // Once per CALL operation & message call transaction. + CreateDataGas uint64 = 200 // + Ripemd160Gas uint64 = 600 // + Ripemd160WordGas uint64 = 120 // + CallCreateDepth uint64 = 1024 // Maximum depth of call/create stack. + ExpGas uint64 = 10 // Once per EXP instruction + LogGas uint64 = 375 // Per LOG* operation. + CopyGas uint64 = 3 // + StackLimit uint64 = 1024 // Maximum size of VM stack allowed. + TierStepGas uint64 = 0 // Once per operation, for a selection of them. + LogTopicGas uint64 = 375 // Multiplied by the * of the LOG*, per LOG transaction. e.g. LOG0 incurs 0 * c_txLogTopicGas, LOG4 incurs 4 * c_txLogTopicGas. + CreateGas uint64 = 32000 // Once per CREATE operation & contract-creation transaction. + SuicideRefundGas uint64 = 24000 // Refunded following a suicide operation. + MemoryGas uint64 = 3 // Times the address of the (highest referenced byte in memory + 1). NOTE: referencing happens on read, write and in instructions such as RETURN and CALL. + TxDataNonZeroGas uint64 = 68 // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. MaxCodeSize = 24576 ) + +var ( + GasLimitBoundDivisor = big.NewInt(1024) // The bound divisor of the gas limit, used in update calculations. + MinGasLimit = big.NewInt(5000) // Minimum the gas limit may ever be. + GenesisGasLimit = big.NewInt(4712388) // Gas limit of the Genesis block. + TargetGasLimit = new(big.Int).Set(GenesisGasLimit) // The artificial target + DifficultyBoundDivisor = big.NewInt(2048) // The bound divisor of the difficulty, used in the update calculations. + GenesisDifficulty = big.NewInt(131072) // Difficulty of the Genesis block. + MinimumDifficulty = big.NewInt(131072) // The minimum that the difficulty may ever be. + DurationLimit = big.NewInt(13) // The decision boundary on the blocktime duration used to determine whether difficulty should go up or not. +) diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 470eb7cb7f..01539de03e 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -146,7 +146,7 @@ func runBlockTests(homesteadBlock, daoForkBlock, gasPriceFork *big.Int, bt map[s } for name, test := range bt { - if skipTest[name] { + if skipTest[name] /*|| name != "CallingCanonicalContractFromFork_CALLCODE"*/ { glog.Infoln("Skipping block test", name) continue } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 5469a4c71b..e8ab29d143 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -108,7 +108,7 @@ func runStateTests(chainConfig *params.ChainConfig, tests map[string]VmTest, ski } for name, test := range tests { - if skipTest[name] { + if skipTest[name] /*|| name != "JUMPDEST_Attack"*/ { glog.Infoln("Skipping state test", name) continue } diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index 25e55886f4..1edf0e425a 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -129,7 +129,7 @@ func runVmTests(tests map[string]VmTest, skipTests []string) error { } for name, test := range tests { - if skipTest[name] /*|| name != "loop_stacklimit_1021"*/ { + if skipTest[name] /*|| name != "exp0"*/ { glog.Infoln("Skipping VM test", name) continue } @@ -229,6 +229,6 @@ func RunVm(statedb *state.StateDB, env, exec map[string]string) ([]byte, []*type vm.PrecompiledContracts = make(map[common.Address]vm.PrecompiledContract) environment, _ := NewEVMEnvironment(true, chainConfig, statedb, env, exec) - ret, err := environment.Call(caller, to, data, gas, value) - return ret, statedb.Logs(), gas, err + ret, g, err := environment.Call(caller, to, data, gas.Uint64(), value) + return ret, statedb.Logs(), new(big.Int).SetUint64(g), err } From 1bed9b3fea9939581b03cae9d6b4984ced456748 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 26 Jan 2017 11:57:31 +0100 Subject: [PATCH 12/21] event: address review issues (multiple commits) event: address Feed review issues event: clarify role of NewSubscription function event: more Feed review fixes * take sendLock after dropping f.mu * add constant for number of special cases event: fix subscribing/unsubscribing while Send is blocked --- event/feed.go | 31 +++++++++++++------- event/feed_test.go | 68 +++++++++++++++++++++++++++++++++++++++++++ event/subscription.go | 10 +++---- 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/event/feed.go b/event/feed.go index bd8e26321d..4568304df5 100644 --- a/event/feed.go +++ b/event/feed.go @@ -33,7 +33,9 @@ var errBadChannel = errors.New("event: Subscribe argument does not have sendable // // The zero value is ready to use. type Feed struct { - sendLock chan struct{} // one-element buffer, empty when held + // sendLock has a one-element buffer and is empty when held. + // It protects sendCases. + sendLock chan struct{} removeSub chan interface{} // interrupts Send sendCases caseList // the active set of select cases used by Send @@ -44,6 +46,10 @@ type Feed struct { closed bool } +// This is the index of the first actual subscription channel in sendCases. +// sendCases[0] is a SelectRecv case for the removeSub channel. +const firstSubSendCase = 1 + type feedTypeError struct { got, want reflect.Type op string @@ -67,6 +73,7 @@ func (f *Feed) init() { // until the subscription is canceled. All channels added must have the same element type. // // The channel should have ample buffer space to avoid blocking other subscribers. +// Slow subscribers are not dropped. func (f *Feed) Subscribe(channel interface{}) Subscription { chanval := reflect.ValueOf(channel) chantyp := chanval.Type() @@ -125,13 +132,14 @@ func (f *Feed) remove(sub *feedSub) { func (f *Feed) Send(value interface{}) (nsent int) { f.mu.Lock() f.init() + f.mu.Unlock() + <-f.sendLock - // Add new subscriptions from the inbox, then clear it. + + // Add new cases from the inbox after taking the send lock. + f.mu.Lock() f.sendCases = append(f.sendCases, f.inbox...) - for i := range f.inbox { - f.inbox[i] = reflect.SelectCase{} - } - f.inbox = f.inbox[:0] + f.inbox = nil f.mu.Unlock() // Set the sent value on all channels. @@ -140,7 +148,7 @@ func (f *Feed) Send(value interface{}) (nsent int) { f.sendLock <- struct{}{} panic(feedTypeError{op: "Send", got: rvalue.Type(), want: f.etype}) } - for i := 1; i < len(f.sendCases); i++ { + for i := firstSubSendCase; i < len(f.sendCases); i++ { f.sendCases[i].Send = rvalue } @@ -150,13 +158,14 @@ func (f *Feed) Send(value interface{}) (nsent int) { // Fast path: try sending without blocking before adding to the select set. // This should usually succeed if subscribers are fast enough and have free // buffer space. - for i := 1; i < len(cases); i++ { + for i := firstSubSendCase; i < len(cases); i++ { if cases[i].Chan.TrySend(rvalue) { - cases = cases.deactivate(i) nsent++ + cases = cases.deactivate(i) + i-- } } - if len(cases) == 1 { + if len(cases) == firstSubSendCase { break } // Select on all the receivers, waiting for them to unblock. @@ -174,7 +183,7 @@ func (f *Feed) Send(value interface{}) (nsent int) { } // Forget about the sent value and hand off the send lock. - for i := 1; i < len(f.sendCases); i++ { + for i := firstSubSendCase; i < len(f.sendCases); i++ { f.sendCases[i].Send = reflect.Value{} } f.sendLock <- struct{}{} diff --git a/event/feed_test.go b/event/feed_test.go index 4f897c162c..a82c103033 100644 --- a/event/feed_test.go +++ b/event/feed_test.go @@ -167,6 +167,74 @@ func TestFeedSubscribeSameChannel(t *testing.T) { done.Wait() } +func TestFeedSubscribeBlockedPost(t *testing.T) { + var ( + feed Feed + nsends = 2000 + ch1 = make(chan int) + ch2 = make(chan int) + wg sync.WaitGroup + ) + defer wg.Wait() + + feed.Subscribe(ch1) + wg.Add(nsends) + for i := 0; i < nsends; i++ { + go func() { + feed.Send(99) + wg.Done() + }() + } + + sub2 := feed.Subscribe(ch2) + defer sub2.Unsubscribe() + + // We're done when ch1 has received N times. + // The number of receives on ch2 depends on scheduling. + for i := 0; i < nsends; { + select { + case <-ch1: + i++ + case <-ch2: + } + } +} + +func TestFeedUnsubscribeBlockedPost(t *testing.T) { + var ( + feed Feed + nsends = 200 + chans = make([]chan int, 2000) + subs = make([]Subscription, len(chans)) + bchan = make(chan int) + bsub = feed.Subscribe(bchan) + wg sync.WaitGroup + ) + for i := range chans { + chans[i] = make(chan int, nsends) + } + + // Queue up some Sends. None of these can make progress while bchan isn't read. + wg.Add(nsends) + for i := 0; i < nsends; i++ { + go func() { + feed.Send(99) + wg.Done() + }() + } + // Subscribe the other channels. + for i, ch := range chans { + subs[i] = feed.Subscribe(ch) + } + // Unsubscribe them again. + for _, sub := range subs { + sub.Unsubscribe() + } + // Unblock the Sends. + bsub.Unsubscribe() + wg.Wait() +} + func TestFeedUnsubscribeFromInbox(t *testing.T) { var ( feed Feed diff --git a/event/subscription.go b/event/subscription.go index 7f2619b2d7..83bd21213d 100644 --- a/event/subscription.go +++ b/event/subscription.go @@ -43,14 +43,14 @@ type Subscription interface { Unsubscribe() // cancels sending of events, closing the error channel } -// NewSubscription runs fn as a subscription in a new goroutine. The channel given to fn -// is closed when Unsubscribe is called. If fn returns an error, it is sent on the -// subscription's error channel. -func NewSubscription(fn func(<-chan struct{}) error) Subscription { +// NewSubscription runs a producer function as a subscription in a new goroutine. The +// channel given to the producer is closed when Unsubscribe is called. If fn returns an +// error, it is sent on the subscription's error channel. +func NewSubscription(producer func(<-chan struct{}) error) Subscription { s := &funcSub{unsub: make(chan struct{}), err: make(chan error, 1)} go func() { defer close(s.err) - err := fn(s.unsub) + err := producer(s.unsub) s.mu.Lock() defer s.mu.Unlock() if !s.unsubscribed { From 9b5c7153c93f077862333efe0217c74ac1514d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 3 Feb 2017 14:04:57 +0200 Subject: [PATCH 13/21] event: use sync.Once for init for faster/cleaner locking --- event/feed.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/event/feed.go b/event/feed.go index 4568304df5..b1b597f17b 100644 --- a/event/feed.go +++ b/event/feed.go @@ -33,9 +33,8 @@ var errBadChannel = errors.New("event: Subscribe argument does not have sendable // // The zero value is ready to use. type Feed struct { - // sendLock has a one-element buffer and is empty when held. - // It protects sendCases. - sendLock chan struct{} + once sync.Once // ensures that init only runs once + sendLock chan struct{} // sendLock has a one-element buffer and is empty when held.It protects sendCases. removeSub chan interface{} // interrupts Send sendCases caseList // the active set of select cases used by Send @@ -60,9 +59,6 @@ func (e feedTypeError) Error() string { } func (f *Feed) init() { - if f.sendLock != nil { - return - } f.removeSub = make(chan interface{}) f.sendLock = make(chan struct{}, 1) f.sendLock <- struct{}{} @@ -75,6 +71,8 @@ func (f *Feed) init() { // The channel should have ample buffer space to avoid blocking other subscribers. // Slow subscribers are not dropped. func (f *Feed) Subscribe(channel interface{}) Subscription { + f.once.Do(f.init) + chanval := reflect.ValueOf(channel) chantyp := chanval.Type() if chantyp.Kind() != reflect.Chan || chantyp.ChanDir()&reflect.SendDir == 0 { @@ -84,7 +82,6 @@ func (f *Feed) Subscribe(channel interface{}) Subscription { f.mu.Lock() defer f.mu.Unlock() - f.init() if !f.typecheck(chantyp.Elem()) { panic(feedTypeError{op: "Subscribe", got: chantyp, want: reflect.ChanOf(reflect.SendDir, f.etype)}) } @@ -130,10 +127,7 @@ func (f *Feed) remove(sub *feedSub) { // Send delivers to all subscribed channels simultaneously. // It returns the number of subscribers that the value was sent to. func (f *Feed) Send(value interface{}) (nsent int) { - f.mu.Lock() - f.init() - f.mu.Unlock() - + f.once.Do(f.init) <-f.sendLock // Add new cases from the inbox after taking the send lock. From 6ea8eba8ce0f203ee99d73f9085939f304ea41c0 Mon Sep 17 00:00:00 2001 From: Diego Siqueira Date: Fri, 3 Feb 2017 10:32:04 -0200 Subject: [PATCH 14/21] accounts/abi, internal/jsre/deps: gofmt -w -s (#3636) Signed-off-by: DiSiqueira --- accounts/abi/type_test.go | 4 ++-- internal/jsre/deps/bindata.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/accounts/abi/type_test.go b/accounts/abi/type_test.go index 1557c2a41b..1558064596 100644 --- a/accounts/abi/type_test.go +++ b/accounts/abi/type_test.go @@ -55,8 +55,8 @@ func TestTypeRegexp(t *testing.T) { {"string[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[]"}}, {"string[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.String, T: StringTy, Size: -1, Elem: &Type{Kind: reflect.String, T: StringTy, Size: -1, stringKind: "string"}, stringKind: "string[2]"}}, {"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}}, - {"address[]", Type{IsSlice: true, SliceSize: -1,Kind: reflect.Array, Type:address_t, T: AddressTy, Size:20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}}, - {"address[2]", Type{IsArray: true, SliceSize: 2,Kind: reflect.Array, Type:address_t, T: AddressTy, Size:20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}}, + {"address[]", Type{IsSlice: true, SliceSize: -1, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}}, + {"address[2]", Type{IsArray: true, SliceSize: 2, Kind: reflect.Array, Type: address_t, T: AddressTy, Size: 20, Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}}, // TODO when fixed types are implemented properly // {"fixed", Type{}}, diff --git a/internal/jsre/deps/bindata.go b/internal/jsre/deps/bindata.go index 73732a4ebc..5f6a2b873a 100644 --- a/internal/jsre/deps/bindata.go +++ b/internal/jsre/deps/bindata.go @@ -206,8 +206,8 @@ type bintree struct { } var _bintree = &bintree{nil, map[string]*bintree{ - "bignumber.js": &bintree{bignumberJs, map[string]*bintree{}}, - "web3.js": &bintree{web3Js, map[string]*bintree{}}, + "bignumber.js": {bignumberJs, map[string]*bintree{}}, + "web3.js": {web3Js, map[string]*bintree{}}, }} // RestoreAsset restores an asset under the given directory From d0eeb3ebdcb58f62ab4c542aef9e865c10797175 Mon Sep 17 00:00:00 2001 From: Bas van Kervel Date: Mon, 6 Feb 2017 18:16:56 +0100 Subject: [PATCH 15/21] cmd/abigen: parse contract name as abi identifier --- cmd/abigen/main.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index dfbd025dad..3a1ae6f4c3 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -94,7 +94,9 @@ func main() { abi, _ := json.Marshal(contract.Info.AbiDefinition) // Flatten the compiler parse abis = append(abis, string(abi)) bins = append(bins, contract.Code) - types = append(types, name) + + nameParts := strings.Split(name, ":") + types = append(types, nameParts[len(nameParts)-1]) } } else { // Otherwise load up the ABI, optional bytecode and type name from the parameters From 6dd27e7cff818493356ddecd8918b69fb18b998a Mon Sep 17 00:00:00 2001 From: Maksim Date: Thu, 9 Feb 2017 00:01:12 +0700 Subject: [PATCH 16/21] swarm/storage: release chunk storage after stop swarm (#3651) closes #3650 --- swarm/storage/dbstore.go | 2 +- swarm/storage/dbstore_test.go | 6 +++--- swarm/storage/dpa.go | 5 +++++ swarm/storage/localstore.go | 5 +++++ swarm/storage/memstore.go | 5 +++++ swarm/storage/netstore.go | 5 +++++ swarm/storage/types.go | 1 + swarm/swarm.go | 16 +++++++++++----- 8 files changed, 36 insertions(+), 9 deletions(-) diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index 4ddebb0217..f5d124d297 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -408,7 +408,7 @@ func (s *DbStore) getEntryCnt() uint64 { return s.entryCnt } -func (s *DbStore) close() { +func (s *DbStore) Close() { s.db.Close() } diff --git a/swarm/storage/dbstore_test.go b/swarm/storage/dbstore_test.go index 3d2b5bc36f..ddce7ccfea 100644 --- a/swarm/storage/dbstore_test.go +++ b/swarm/storage/dbstore_test.go @@ -38,7 +38,7 @@ func initDbStore(t *testing.T) *DbStore { func testDbStore(l int64, branches int64, t *testing.T) { m := initDbStore(t) - defer m.close() + defer m.Close() testStore(m, l, branches, t) } @@ -64,7 +64,7 @@ func TestDbStore2_100_(t *testing.T) { func TestDbStoreNotFound(t *testing.T) { m := initDbStore(t) - defer m.close() + defer m.Close() _, err := m.Get(ZeroKey) if err != notFound { t.Errorf("Expected notFound, got %v", err) @@ -73,7 +73,7 @@ func TestDbStoreNotFound(t *testing.T) { func TestDbStoreSyncIterator(t *testing.T) { m := initDbStore(t) - defer m.close() + defer m.Close() keys := []Key{ Key(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")), Key(common.Hex2Bytes("4000000000000000000000000000000000000000000000000000000000000000")), diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 31b6c54ac2..7b3e23cac2 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -237,3 +237,8 @@ func (self *dpaChunkStore) Put(entry *Chunk) { self.n++ self.netStore.Put(chunk) } + +// Close chunk store +func (self *dpaChunkStore) Close() { + return +} diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 541462f0c4..14827e3618 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -72,3 +72,8 @@ func (self *LocalStore) Get(key Key) (chunk *Chunk, err error) { self.memStore.Put(chunk) return } + +// Close local store +func (self *LocalStore) Close() { + return +} diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index f133bd7d31..e55abb45fc 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -314,3 +314,8 @@ func (s *MemStore) removeOldest() { } } } + +// Close memstore +func (s *MemStore) Close() { + return +} diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index f97862bbbd..46479b58a5 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -132,3 +132,8 @@ func (self *NetStore) Get(key Key) (*Chunk, error) { go self.cloud.Retrieve(chunk) return chunk, nil } + +// Close netstore +func (self *NetStore) Close() { + return +} diff --git a/swarm/storage/types.go b/swarm/storage/types.go index f3ab99c6c8..cc5ded931d 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -167,6 +167,7 @@ The ChunkStore interface is implemented by : type ChunkStore interface { Put(*Chunk) // effectively there is no error even if there is an error Get(Key) (*Chunk, error) + Close() } /* diff --git a/swarm/swarm.go b/swarm/swarm.go index 4b3621affc..eab01f036c 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -54,6 +54,7 @@ type Swarm struct { privateKey *ecdsa.PrivateKey corsString string swapEnabled bool + lstore *storage.LocalStore // local store, needs to store for releasing resources after node stopped } type SwarmAPI struct { @@ -90,7 +91,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. glog.V(logger.Debug).Infof("Setting up Swarm service components") hash := storage.MakeHashFunc(config.ChunkerParams.Hash) - lstore, err := storage.NewLocalStore(hash, config.StoreParams) + self.lstore, err = storage.NewLocalStore(hash, config.StoreParams) if err != nil { return } @@ -98,7 +99,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. // setup local store glog.V(logger.Debug).Infof("Set up local storage") - self.dbAccess = network.NewDbAccess(lstore) + self.dbAccess = network.NewDbAccess(self.lstore) glog.V(logger.Debug).Infof("Set up local db access (iterator/counter)") // set up the kademlia hive @@ -115,15 +116,15 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. glog.V(logger.Debug).Infof("-> set swarm forwarder as cloud storage backend") // setup cloud storage internal access layer - self.storage = storage.NewNetStore(hash, lstore, cloud, config.StoreParams) + self.storage = storage.NewNetStore(hash, self.lstore, cloud, config.StoreParams) glog.V(logger.Debug).Infof("-> swarm net store shared access layer to Swarm Chunk Store") // set up Depo (storage handler = cloud storage access layer for incoming remote requests) - self.depo = network.NewDepo(hash, lstore, self.storage) + self.depo = network.NewDepo(hash, self.lstore, self.storage) glog.V(logger.Debug).Infof("-> REmote Access to CHunks") // set up DPA, the cloud storage local access layer - dpaChunkStore := storage.NewDpaChunkStore(lstore, self.storage) + dpaChunkStore := storage.NewDpaChunkStore(self.lstore, self.storage) glog.V(logger.Debug).Infof("-> Local Access to Swarm") // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage self.dpa = storage.NewDPA(dpaChunkStore, self.config.ChunkerParams) @@ -212,6 +213,11 @@ func (self *Swarm) Stop() error { ch.Stop() ch.Save() } + + if self.lstore != nil { + self.lstore.DbStore.Close() + } + return self.config.Save() } From 7ff686d6eca17e3d923ae4ccc479166e225a0b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 10 Feb 2017 17:14:56 +0200 Subject: [PATCH 17/21] travis: split Android off OSX, use native image --- .travis.yml | 51 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3384f3ec66..a86e32bba0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,27 +53,52 @@ matrix: - CC=aarch64-linux-gnu-gcc go run build/ci.go install -arch arm64 - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - # This builder does the OSX Azure, Android Maven and Azure and iOS CocoaPods and Azure uploads + # This builder does the Android Maven and Azure uploads + - os: linux + dist: precise # Needed for the android tools + addons: + apt: + packages: + - oracle-java8-installer + - oracle-java8-set-default + language: android + android: + components: + - platform-tools + - tools + - android-15 + - android-19 + - android-24 + env: + - azure-android + - maven-android + before_install: + - curl https://storage.googleapis.com/golang/go1.8rc3.linux-amd64.tar.gz | tar -xz + - export PATH=`pwd`/go/bin:$PATH + - export GOROOT=`pwd`/go + - export GOPATH=$HOME/go # Drop post Go 1.8 + script: + # Build the Android archive and upload it to Maven Central and Azure + - curl https://dl.google.com/android/repository/android-ndk-r13b-linux-x86_64.zip -o android-ndk-r13b.zip + - unzip -q android-ndk-r13b.zip && rm android-ndk-r13b.zip + - mv android-ndk-r13b $HOME + - export ANDROID_NDK=$HOME/android-ndk-r13b + + - mkdir -p $GOPATH/src/github.com/ethereum + - ln -s `pwd` $GOPATH/src/github.com/ethereum + - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds + + # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads - os: osx go: 1.7.4 env: - azure-osx - - mobile + - azure-ios + - cocoapods-ios script: - go run build/ci.go install - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -upload gethstore/builds - # Build the Android archive and upload it to Maven Central and Azure - - brew update - - travis_wait 60 brew install android-sdk android-ndk maven gpg - - alias gpg="gpg2" - - - export ANDROID_HOME=/usr/local/opt/android-sdk - - export ANDROID_NDK=/usr/local/opt/android-ndk - - echo "y" | android update sdk --no-ui --filter `android list sdk | grep "SDK Platform Android" | grep -E 'API 15|API 19|API 24' | awk '{print $1}' | cut -d '-' -f 1 | tr '\n' ','` - - - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds - # Build the iOS framework and upload it to CocoaPods and Azure - gem uninstall cocoapods -a - gem install cocoapods --pre From 085987ff2cb1695881a3d0abeebbc21a4c51a2de Mon Sep 17 00:00:00 2001 From: Zahoor Mohamed Date: Mon, 13 Feb 2017 08:03:05 +0530 Subject: [PATCH 18/21] cmd/swarm: manifest manipulation commands (#3645) --- cmd/swarm/main.go | 37 +++++ cmd/swarm/manifest.go | 360 ++++++++++++++++++++++++++++++++++++++++++ cmd/swarm/upload.go | 26 +++ 3 files changed, 423 insertions(+) create mode 100644 cmd/swarm/manifest.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 34b3f71c4e..546c646f11 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -154,6 +154,43 @@ The output of this command is supposed to be machine-readable. Prints the swarm hash of file or directory. `, }, + { + Name: "manifest", + Usage: "update a MANIFEST", + ArgsUsage: "manifest COMMAND", + Description: ` +Updates a MANIFEST by adding/removing/updating the hash of a path. +`, + Subcommands: []cli.Command{ + { + Action: add, + Name: "add", + Usage: "add a new path to the manifest", + ArgsUsage: " []", + Description: ` +Adds a new path to the manifest +`, + }, + { + Action: update, + Name: "update", + Usage: "update the hash for an already existing path in the manifest", + ArgsUsage: " []", + Description: ` +Update the hash for an already existing path in the manifest +`, + }, + { + Action: remove, + Name: "remove", + Usage: "removes a path from the manifest", + ArgsUsage: " ", + Description: ` +Removes a path from the manifest +`, + }, + }, + }, } app.Flags = []cli.Flag{ diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go new file mode 100644 index 0000000000..0de0d69bbf --- /dev/null +++ b/cmd/swarm/manifest.go @@ -0,0 +1,360 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// Command MANIFEST update +package main + +import ( + "gopkg.in/urfave/cli.v1" + "log" + "mime" + "path/filepath" + "strings" + "fmt" + "encoding/json" +) + +func add(ctx *cli.Context) { + + args := ctx.Args() + if len(args) < 3 { + log.Fatal("need atleast three arguments []") + } + + var ( + mhash = args[0] + path = args[1] + hash = args[2] + + ctype string + wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) + mroot manifest + ) + + + if len(args) > 3 { + ctype = args[3] + } else { + ctype = mime.TypeByExtension(filepath.Ext(path)) + } + + newManifest := addEntryToManifest (ctx, mhash, path, hash, ctype) + fmt.Println(newManifest) + + if !wantManifest { + // Print the manifest. This is the only output to stdout. + mrootJSON, _ := json.MarshalIndent(mroot, "", " ") + fmt.Println(string(mrootJSON)) + return + } +} + +func update(ctx *cli.Context) { + + args := ctx.Args() + if len(args) < 3 { + log.Fatal("need atleast three arguments ") + } + + var ( + mhash = args[0] + path = args[1] + hash = args[2] + + ctype string + wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) + mroot manifest + ) + if len(args) > 3 { + ctype = args[3] + } else { + ctype = mime.TypeByExtension(filepath.Ext(path)) + } + + newManifest := updateEntryInManifest (ctx, mhash, path, hash, ctype) + fmt.Println(newManifest) + + if !wantManifest { + // Print the manifest. This is the only output to stdout. + mrootJSON, _ := json.MarshalIndent(mroot, "", " ") + fmt.Println(string(mrootJSON)) + return + } +} + +func remove(ctx *cli.Context) { + args := ctx.Args() + if len(args) < 2 { + log.Fatal("need atleast two arguments ") + } + + var ( + mhash = args[0] + path = args[1] + + wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) + mroot manifest + ) + + newManifest := removeEntryFromManifest (ctx, mhash, path) + fmt.Println(newManifest) + + if !wantManifest { + // Print the manifest. This is the only output to stdout. + mrootJSON, _ := json.MarshalIndent(mroot, "", " ") + fmt.Println(string(mrootJSON)) + return + } +} + +func addEntryToManifest(ctx *cli.Context, mhash , path, hash , ctype string) string { + + var ( + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client = &client{api: bzzapi} + longestPathEntry = manifestEntry{ + Path: "", + Hash: "", + ContentType: "", + } + ) + + mroot, err := client.downloadManifest(mhash) + if err != nil { + log.Fatalln("manifest download failed:", err) + } + + //TODO: check if the "hash" to add is valid and present in swarm + _, err = client.downloadManifest(hash) + if err != nil { + log.Fatalln("hash to add is not present:", err) + } + + + // See if we path is in this Manifest or do we have to dig deeper + for _, entry := range mroot.Entries { + if path == entry.Path { + log.Fatal(path, "Already present, not adding anything") + }else { + if entry.ContentType == "application/bzz-manifest+json" { + prfxlen := strings.HasPrefix(path, entry.Path) + if prfxlen && len(path) > len(longestPathEntry.Path) { + longestPathEntry = entry + } + } + } + } + + if longestPathEntry.Path != "" { + // Load the child Manifest add the entry there + newPath := path[len(longestPathEntry.Path):] + newHash := addEntryToManifest (ctx, longestPathEntry.Hash, newPath, hash, ctype) + + // Replace the hash for parent Manifests + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if longestPathEntry.Path == entry.Path { + entry.Hash = newHash + } + newMRoot.Entries = append(newMRoot.Entries, entry) + } + mroot = newMRoot + } else { + // Add the entry in the leaf Manifest + newEntry := manifestEntry{ + Path: path, + Hash: hash, + ContentType: ctype, + } + mroot.Entries = append(mroot.Entries, newEntry) + } + + + newManifestHash, err := client.uploadManifest(mroot) + if err != nil { + log.Fatalln("manifest upload failed:", err) + } + return newManifestHash + + + +} + +func updateEntryInManifest(ctx *cli.Context, mhash , path, hash , ctype string) string { + + var ( + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client = &client{api: bzzapi} + newEntry = manifestEntry{ + Path: "", + Hash: "", + ContentType: "", + } + longestPathEntry = manifestEntry{ + Path: "", + Hash: "", + ContentType: "", + } + ) + + mroot, err := client.downloadManifest(mhash) + if err != nil { + log.Fatalln("manifest download failed:", err) + } + + //TODO: check if the "hash" with which to update is valid and present in swarm + + + // See if we path is in this Manifest or do we have to dig deeper + for _, entry := range mroot.Entries { + if path == entry.Path { + newEntry = entry + }else { + if entry.ContentType == "application/bzz-manifest+json" { + prfxlen := strings.HasPrefix(path, entry.Path) + if prfxlen && len(path) > len(longestPathEntry.Path) { + longestPathEntry = entry + } + } + } + } + + if longestPathEntry.Path == "" && newEntry.Path == "" { + log.Fatal(path, " Path not present in the Manifest, not setting anything") + } + + if longestPathEntry.Path != "" { + // Load the child Manifest add the entry there + newPath := path[len(longestPathEntry.Path):] + newHash := updateEntryInManifest (ctx, longestPathEntry.Hash, newPath, hash, ctype) + + // Replace the hash for parent Manifests + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if longestPathEntry.Path == entry.Path { + entry.Hash = newHash + } + newMRoot.Entries = append(newMRoot.Entries, entry) + + } + mroot = newMRoot + } + + if newEntry.Path != "" { + // Replace the hash for leaf Manifest + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if newEntry.Path == entry.Path { + myEntry := manifestEntry{ + Path: entry.Path, + Hash: hash, + ContentType: ctype, + } + newMRoot.Entries = append(newMRoot.Entries, myEntry) + } else { + newMRoot.Entries = append(newMRoot.Entries, entry) + } + } + mroot = newMRoot + } + + + newManifestHash, err := client.uploadManifest(mroot) + if err != nil { + log.Fatalln("manifest upload failed:", err) + } + return newManifestHash +} + +func removeEntryFromManifest(ctx *cli.Context, mhash , path string) string { + + var ( + bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client = &client{api: bzzapi} + entryToRemove = manifestEntry{ + Path: "", + Hash: "", + ContentType: "", + } + longestPathEntry = manifestEntry{ + Path: "", + Hash: "", + ContentType: "", + } + ) + + mroot, err := client.downloadManifest(mhash) + if err != nil { + log.Fatalln("manifest download failed:", err) + } + + + + // See if we path is in this Manifest or do we have to dig deeper + for _, entry := range mroot.Entries { + if path == entry.Path { + entryToRemove = entry + }else { + if entry.ContentType == "application/bzz-manifest+json" { + prfxlen := strings.HasPrefix(path, entry.Path) + if prfxlen && len(path) > len(longestPathEntry.Path) { + longestPathEntry = entry + } + } + } + } + + if longestPathEntry.Path == "" && entryToRemove.Path == "" { + log.Fatal(path, "Path not present in the Manifest, not removing anything") + } + + if longestPathEntry.Path != "" { + // Load the child Manifest remove the entry there + newPath := path[len(longestPathEntry.Path):] + newHash := removeEntryFromManifest (ctx, longestPathEntry.Hash, newPath) + + // Replace the hash for parent Manifests + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if longestPathEntry.Path == entry.Path { + entry.Hash = newHash + } + newMRoot.Entries = append(newMRoot.Entries, entry) + } + mroot = newMRoot + } + + if entryToRemove.Path != "" { + // remove the entry in this Manifest + newMRoot := manifest{} + for _, entry := range mroot.Entries { + if entryToRemove.Path != entry.Path { + newMRoot.Entries = append(newMRoot.Entries, entry) + } + } + mroot = newMRoot + } + + + newManifestHash, err := client.uploadManifest(mroot) + if err != nil { + log.Fatalln("manifest upload failed:", err) + } + return newManifestHash + + +} + diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index d8039d45b5..871713b2da 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -229,3 +229,29 @@ func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (strin content, err := ioutil.ReadAll(resp.Body) return string(content), err } + +func (c *client) downloadManifest(mhash string) (manifest, error) { + + mroot := manifest{} + req, err := http.NewRequest("GET", c.api + "/bzzr:/" + mhash, nil) + if err != nil { + return mroot, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return mroot, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + return mroot, fmt.Errorf("bad status: %s", resp.Status) + + } + content, err := ioutil.ReadAll(resp.Body) + + err = json.Unmarshal(content, &mroot) + if err != nil { + return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err) + } + return mroot, err +} \ No newline at end of file From 564b60520c68a1f06171abd705c01946b932492f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 13 Feb 2017 03:36:50 +0100 Subject: [PATCH 19/21] core: ignore 0x prefix for code in JSON genesis blocks (#3656) --- core/genesis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/genesis.go b/core/genesis.go index a06c404082..b94b5af76f 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -70,7 +70,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, for addr, account := range genesis.Alloc { address := common.HexToAddress(addr) statedb.AddBalance(address, common.String2Big(account.Balance)) - statedb.SetCode(address, common.Hex2Bytes(account.Code)) + statedb.SetCode(address, common.FromHex(account.Code)) statedb.SetNonce(address, common.String2Big(account.Nonce).Uint64()) for key, value := range account.Storage { statedb.SetState(address, common.HexToHash(key), common.HexToHash(value)) From 65ed6a9def4d23bbe6109ae4b841a56510d0c476 Mon Sep 17 00:00:00 2001 From: gluk256 Date: Mon, 13 Feb 2017 13:15:20 +0100 Subject: [PATCH 20/21] whisper: add tests for mailserver (#3631) --- cmd/wnode/main.go | 7 +- whisper/mailserver/mailserver.go | 36 ++++-- whisper/mailserver/server_test.go | 183 ++++++++++++++++++++++++++++++ 3 files changed, 217 insertions(+), 9 deletions(-) create mode 100644 whisper/mailserver/server_test.go diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index cbf093aa7e..55565eab26 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -209,10 +209,15 @@ func initialize() { nodeid = shh.NewIdentity() } + maxPeers := 80 + if *bootstrapMode { + maxPeers = 800 + } + server = &p2p.Server{ Config: p2p.Config{ PrivateKey: nodeid, - MaxPeers: 128, + MaxPeers: maxPeers, Name: common.MakeName("whisper-go", "5.0"), Protocols: shh.Protocols(), ListenAddr: *argIP, diff --git a/whisper/mailserver/mailserver.go b/whisper/mailserver/mailserver.go index f7d6c3e5c1..3e08a3b7e2 100644 --- a/whisper/mailserver/mailserver.go +++ b/whisper/mailserver/mailserver.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/rlp" @@ -101,11 +102,19 @@ func (s *WMailServer) Archive(env *whisper.Envelope) { } func (s *WMailServer) DeliverMail(peer *whisper.Peer, request *whisper.Envelope) { - ok, lower, upper, topic := s.validate(peer, request) - if !ok { + if peer == nil { + glog.V(logger.Error).Info("Whisper peer is nil") return } + ok, lower, upper, topic := s.validateRequest(peer.ID(), request) + if ok { + s.processRequest(peer, lower, upper, topic) + } +} + +func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, topic whisper.TopicType) []*whisper.Envelope { + ret := make([]*whisper.Envelope, 0) var err error var zero common.Hash var empty whisper.TopicType @@ -122,10 +131,15 @@ func (s *WMailServer) DeliverMail(peer *whisper.Peer, request *whisper.Envelope) } if topic == empty || envelope.Topic == topic { - err = s.w.SendP2PDirect(peer, &envelope) - if err != nil { - glog.V(logger.Error).Infof("Failed to send direct message to peer: %s", err) - return + if peer == nil { + // used for test purposes + ret = append(ret, &envelope) + } else { + err = s.w.SendP2PDirect(peer, &envelope) + if err != nil { + glog.V(logger.Error).Infof("Failed to send direct message to peer: %s", err) + return nil + } } } } @@ -134,9 +148,11 @@ func (s *WMailServer) DeliverMail(peer *whisper.Peer, request *whisper.Envelope) if err != nil { glog.V(logger.Error).Infof("Level DB iterator error: %s", err) } + + return ret } -func (s *WMailServer) validate(peer *whisper.Peer, request *whisper.Envelope) (bool, uint32, uint32, whisper.TopicType) { +func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) (bool, uint32, uint32, whisper.TopicType) { var topic whisper.TopicType if s.pow > 0.0 && request.PoW() < s.pow { return false, 0, 0, topic @@ -154,7 +170,11 @@ func (s *WMailServer) validate(peer *whisper.Peer, request *whisper.Envelope) (b return false, 0, 0, topic } - if bytes.Equal(peer.ID(), decrypted.Signature) { + src := crypto.FromECDSAPub(decrypted.Src) + if len(src)-len(peerID) == 1 { + src = src[1:] + } + if !bytes.Equal(peerID, src) { glog.V(logger.Warn).Infof("Wrong signature of p2p request") return false, 0, 0, topic } diff --git a/whisper/mailserver/server_test.go b/whisper/mailserver/server_test.go new file mode 100644 index 0000000000..24abf6c1ac --- /dev/null +++ b/whisper/mailserver/server_test.go @@ -0,0 +1,183 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package mailserver + +import ( + "crypto/ecdsa" + "encoding/binary" + "io/ioutil" + "math/rand" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" +) + +const powRequirement = 0.00001 +const keyName = "6d604bac5401ce9a6b995f1b45a4ab" + +var shh *whisper.Whisper +var seed = time.Now().Unix() + +type ServerTestParams struct { + topic whisper.TopicType + low uint32 + upp uint32 + key *ecdsa.PrivateKey +} + +func assert(statement bool, text string, t *testing.T) { + if !statement { + t.Fatal(text) + } +} + +func TestDBKey(t *testing.T) { + var h common.Hash + i := uint32(time.Now().Unix()) + k := NewDbKey(i, h) + assert(len(k.raw) == common.HashLength+4, "wrong DB key length", t) + assert(byte(i%0x100) == k.raw[3], "raw representation should be big endian", t) + assert(byte(i/0x1000000) == k.raw[0], "big endian expected", t) +} + +func generateEnvelope(t *testing.T) *whisper.Envelope { + params := &whisper.MessageParams{ + KeySym: []byte("test key"), + Topic: whisper.TopicType{}, + Payload: []byte("test payload"), + PoW: powRequirement, + WorkTime: 2, + } + + msg := whisper.NewSentMessage(params) + env, err := msg.Wrap(params) + if err != nil { + t.Fatalf("failed to wrap with seed %d: %s.", seed, err) + } + return env +} + +func TestMailServer(t *testing.T) { + const password = "password_for_this_test" + const dbPath = "whisper-server-test" + + _, err := ioutil.TempDir("", dbPath) + if err != nil { + t.Fatal(err) + } + + var server WMailServer + shh = whisper.NewWhisper(&server) + server.Init(shh, dbPath, password, powRequirement) + defer server.Close() + + err = shh.AddSymKey(keyName, []byte(password)) + if err != nil { + t.Fatalf("Failed to create symmetric key for mail request: %s", err) + } + + rand.Seed(seed) + env := generateEnvelope(t) + server.Archive(env) + deliverTest(t, &server, env) +} + +func deliverTest(t *testing.T, server *WMailServer, env *whisper.Envelope) { + testPeerID := shh.NewIdentity() + birth := env.Expiry - env.TTL + p := &ServerTestParams{ + topic: env.Topic, + low: birth - 1, + upp: birth + 1, + key: testPeerID, + } + singleRequest(t, server, env, p, true) + + p.low, p.upp = birth+1, 0xffffffff + singleRequest(t, server, env, p, false) + + p.low, p.upp = 0, birth-1 + singleRequest(t, server, env, p, false) + + p.low = birth - 1 + p.upp = birth + 1 + p.topic[0]++ + singleRequest(t, server, env, p, false) +} + +func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p *ServerTestParams, expect bool) { + request := createRequest(t, p) + src := crypto.FromECDSAPub(&p.key.PublicKey) + ok, lower, upper, topic := server.validateRequest(src, request) + if !ok { + t.Fatalf("request validation failed, seed: %d.", seed) + } + if lower != p.low { + t.Fatalf("request validation failed (lower bound), seed: %d.", seed) + } + if upper != p.upp { + t.Fatalf("request validation failed (upper bound), seed: %d.", seed) + } + if topic != p.topic { + t.Fatalf("request validation failed (topic), seed: %d.", seed) + } + + var exist bool + mail := server.processRequest(nil, p.low, p.upp, p.topic) + for _, msg := range mail { + if msg.Hash() == env.Hash() { + exist = true + break + } + } + + if exist != expect { + t.Fatalf("error: exist = %v, seed: %d.", exist, seed) + } + + src[0]++ + ok, lower, upper, topic = server.validateRequest(src, request) + if ok { + t.Fatalf("request validation false positive, seed: %d.", seed) + } +} + +func createRequest(t *testing.T, p *ServerTestParams) *whisper.Envelope { + data := make([]byte, 8+whisper.TopicLength) + binary.BigEndian.PutUint32(data, p.low) + binary.BigEndian.PutUint32(data[4:], p.upp) + copy(data[8:], p.topic[:]) + + params := &whisper.MessageParams{ + KeySym: shh.GetSymKey(keyName), + Topic: p.topic, + Payload: data, + PoW: powRequirement * 2, + WorkTime: 2, + Src: p.key, + } + + msg := whisper.NewSentMessage(params) + env, err := msg.Wrap(params) + if err != nil { + t.Fatalf("failed to wrap with seed %d: %s.", seed, err) + } + return env +} From e23e86921b55cb1ee2fca6b6fb9ed91f5532f9fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Mon, 13 Feb 2017 18:50:50 +0630 Subject: [PATCH 21/21] swarm/network: fix chunk integrity checks (#3665) * swarm/network: integrity on incoming known chunks * swarm/network: fix integrity check for incoming chunks * swarm/storage: imrpoved integrity checking on chunks * dbstore panics on corrupt chunk entry an prompts user to run cleandb * memstore adds logging for garbage collection * dbstore refactor item delete. correct partial deletes in Get * cmd/swarm: added cleandb subcommand --- cmd/swarm/cleandb.go | 39 +++++++++++++++++++++++++ cmd/swarm/main.go | 9 ++++++ swarm/network/depo.go | 24 +++++++++------ swarm/network/syncer.go | 6 ++-- swarm/storage/dbstore.go | 61 +++++++++++++++++++++++++++++++++------ swarm/storage/memstore.go | 7 +++++ 6 files changed, 125 insertions(+), 21 deletions(-) create mode 100644 cmd/swarm/cleandb.go diff --git a/cmd/swarm/cleandb.go b/cmd/swarm/cleandb.go new file mode 100644 index 0000000000..81636ada5d --- /dev/null +++ b/cmd/swarm/cleandb.go @@ -0,0 +1,39 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "log" + + "github.com/ethereum/go-ethereum/swarm/storage" + "gopkg.in/urfave/cli.v1" +) + +func cleandb(ctx *cli.Context) { + args := ctx.Args() + if len(args) != 1 { + log.Fatal("need path to chunks database as the first and only argument") + } + + chunkDbPath := args[0] + hash := storage.MakeHashFunc("SHA3") + dbStore, err := storage.NewDbStore(chunkDbPath, hash, 10000000, 0) + if err != nil { + log.Fatalf("cannot initialise dbstore: %v", err) + } + dbStore.Cleanup() +} diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 546c646f11..7d76d55c1f 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -191,6 +191,15 @@ Removes a path from the manifest }, }, }, + { + Action: cleandb, + Name: "cleandb", + Usage: "Cleans database of corrupted entries", + ArgsUsage: " ", + Description: ` +Cleans database of corrupted entries. +`, + }, } app.Flags = []cli.Flag{ diff --git a/swarm/network/depo.go b/swarm/network/depo.go index 79987cc6be..454a572709 100644 --- a/swarm/network/depo.go +++ b/swarm/network/depo.go @@ -99,6 +99,7 @@ func (self *Depo) HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer) // if key found locally, return. otherwise // remote is untrusted, so hash is verified and chunk passed on to NetStore func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) { + var islocal bool req.from = p chunk, err := self.localStore.Get(req.Key) switch { @@ -110,27 +111,32 @@ func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) { case chunk.SData == nil: // found chunk in memory store, needs the data, validate now - hasher := self.hashfunc() - hasher.Write(req.SData) - if !bytes.Equal(hasher.Sum(nil), req.Key) { - // data does not validate, ignore - // TODO: peer should be penalised/dropped? - glog.V(logger.Warn).Infof("Depo.HandleStoreRequest: chunk invalid. store request ignored: %v", req) - return - } glog.V(logger.Detail).Infof("Depo.HandleStoreRequest: %v. request entry found", req) default: // data is found, store request ignored // this should update access count? glog.V(logger.Detail).Infof("Depo.HandleStoreRequest: %v found locally. ignore.", req) + islocal = true + //return + } + + hasher := self.hashfunc() + hasher.Write(req.SData) + if !bytes.Equal(hasher.Sum(nil), req.Key) { + // data does not validate, ignore + // TODO: peer should be penalised/dropped? + glog.V(logger.Warn).Infof("Depo.HandleStoreRequest: chunk invalid. store request ignored: %v", req) return } + if islocal { + return + } // update chunk with size and data chunk.SData = req.SData // protocol validates that SData is minimum 9 bytes long (int64 size + at least one byte of data) chunk.Size = int64(binary.LittleEndian.Uint64(req.SData[0:8])) - glog.V(logger.Detail).Infof("delivery of %p from %v", chunk, p) + glog.V(logger.Detail).Infof("delivery of %v from %v", chunk, p) chunk.Source = p self.netStore.Put(chunk) } diff --git a/swarm/network/syncer.go b/swarm/network/syncer.go index e871666bd1..b6b1ea3b60 100644 --- a/swarm/network/syncer.go +++ b/swarm/network/syncer.go @@ -438,7 +438,7 @@ LOOP: for priority = High; priority >= 0; priority-- { // the first priority channel that is non-empty will be assigned to keys if len(self.keys[priority]) > 0 { - glog.V(logger.Detail).Infof("syncer[%v]: reading request with priority %v", self.key.Log(), priority) + glog.V(logger.Detail).Infof("syncer[%v]: reading request with priority %v", self.key.Log(), priority) keys = self.keys[priority] break PRIORITIES } @@ -551,10 +551,10 @@ LOOP: } if sreq, err := self.newSyncRequest(req, priority); err == nil { // extract key from req - glog.V(logger.Detail).Infof("syncer(priority %v): request %v (synced = %v)", self.key.Log(), priority, req, state.Synced) + glog.V(logger.Detail).Infof("syncer[%v]: (priority %v): request %v (synced = %v)", self.key.Log(), priority, req, state.Synced) unsynced = append(unsynced, sreq) } else { - glog.V(logger.Warn).Infof("syncer(priority %v): error creating request for %v: %v)", self.key.Log(), priority, req, state.Synced, err) + glog.V(logger.Warn).Infof("syncer[%v]: (priority %v): error creating request for %v: %v)", self.key.Log(), priority, req, state.Synced, err) } } diff --git a/swarm/storage/dbstore.go b/swarm/storage/dbstore.go index f5d124d297..e320cd3279 100644 --- a/swarm/storage/dbstore.go +++ b/swarm/storage/dbstore.go @@ -252,12 +252,7 @@ func (s *DbStore) collectGarbage(ratio float32) { // actual gc for i := 0; i < gcnt; i++ { if s.gcArray[i].value <= cutval { - batch := new(leveldb.Batch) - batch.Delete(s.gcArray[i].idxKey) - batch.Delete(getDataKey(s.gcArray[i].idx)) - s.entryCnt-- - batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) - s.db.Write(batch) + s.delete(s.gcArray[i].idx, s.gcArray[i].idxKey) } } @@ -266,6 +261,52 @@ func (s *DbStore) collectGarbage(ratio float32) { s.db.Put(keyGCPos, s.gcPos) } +func (s *DbStore) Cleanup() { + //Iterates over the database and checks that there are no faulty chunks + it := s.db.NewIterator() + startPosition := []byte{kpIndex} + it.Seek(startPosition) + var key []byte + var errorsFound, total int + for it.Valid() { + key = it.Key() + if (key == nil) || (key[0] != kpIndex) { + break + } + total++ + var index dpaDBIndex + decodeIndex(it.Value(), &index) + + data, err := s.db.Get(getDataKey(index.Idx)) + if err != nil { + glog.V(logger.Warn).Infof("Chunk %x found but could not be accessed: %v", key[:], err) + s.delete(index.Idx, getIndexKey(key[1:])) + errorsFound++ + } else { + hasher := s.hashfunc() + hasher.Write(data) + hash := hasher.Sum(nil) + if !bytes.Equal(hash, key[1:]) { + glog.V(logger.Warn).Infof("Found invalid chunk. Hash mismatch. hash=%x, key=%x", hash, key[:]) + s.delete(index.Idx, getIndexKey(key[1:])) + errorsFound++ + } + } + it.Next() + } + it.Release() + glog.V(logger.Warn).Infof("Found %v errors out of %v entries", errorsFound, total) +} + +func (s *DbStore) delete(idx uint64, idxKey []byte) { + batch := new(leveldb.Batch) + batch.Delete(idxKey) + batch.Delete(getDataKey(idx)) + s.entryCnt-- + batch.Put(keyEntryCnt, U64ToBytes(s.entryCnt)) + s.db.Write(batch) +} + func (s *DbStore) Counter() uint64 { s.lock.Lock() defer s.lock.Unlock() @@ -283,6 +324,7 @@ func (s *DbStore) Put(chunk *Chunk) { if chunk.dbStored != nil { close(chunk.dbStored) } + glog.V(logger.Detail).Infof("Storing to DB: chunk already exists, only update access") return // already exists, only update access } @@ -348,6 +390,8 @@ func (s *DbStore) Get(key Key) (chunk *Chunk, err error) { var data []byte data, err = s.db.Get(getDataKey(index.Idx)) if err != nil { + glog.V(logger.Detail).Infof("DBStore: Chunk %v found but could not be accessed: %v", key.Log(), err) + s.delete(index.Idx, getIndexKey(key)) return } @@ -355,9 +399,8 @@ func (s *DbStore) Get(key Key) (chunk *Chunk, err error) { hasher.Write(data) hash := hasher.Sum(nil) if !bytes.Equal(hash, key) { - s.db.Delete(getDataKey(index.Idx)) - err = fmt.Errorf("invalid chunk. hash=%x, key=%v", hash, key[:]) - return + s.delete(index.Idx, getIndexKey(key)) + panic("Invalid Chunk in Database. Please repair with command: 'swarm cleandb'") } chunk = &Chunk{ diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index e55abb45fc..7903d33e70 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -20,6 +20,9 @@ package storage import ( "sync" + + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" ) const ( @@ -284,7 +287,11 @@ func (s *MemStore) removeOldest() { } if node.entry.dbStored != nil { + glog.V(logger.Detail).Infof("Memstore Clean: Waiting for chunk %v to be saved", node.entry.Key.Log()) <-node.entry.dbStored + glog.V(logger.Detail).Infof("Memstore Clean: Chunk %v saved to DBStore. Ready to clear from mem.", node.entry.Key.Log()) + } else { + glog.V(logger.Detail).Infof("Memstore Clean: Chunk %v already in DB. Ready to delete.", node.entry.Key.Log()) } if node.entry.SData != nil {