diff --git a/Godeps/_workspace/src/golang.org/x/net/context/context.go b/Godeps/_workspace/src/golang.org/x/net/context/context.go new file mode 100644 index 0000000000..e7ee376c47 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/context/context.go @@ -0,0 +1,447 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package context defines the Context type, which carries deadlines, +// cancelation signals, and other request-scoped values across API boundaries +// and between processes. +// +// Incoming requests to a server should create a Context, and outgoing calls to +// servers should accept a Context. The chain of function calls between must +// propagate the Context, optionally replacing it with a modified copy created +// using WithDeadline, WithTimeout, WithCancel, or WithValue. +// +// Programs that use Contexts should follow these rules to keep interfaces +// consistent across packages and enable static analysis tools to check context +// propagation: +// +// Do not store Contexts inside a struct type; instead, pass a Context +// explicitly to each function that needs it. The Context should be the first +// parameter, typically named ctx: +// +// func DoSomething(ctx context.Context, arg Arg) error { +// // ... use ctx ... +// } +// +// Do not pass a nil Context, even if a function permits it. Pass context.TODO +// if you are unsure about which Context to use. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +// +// The same Context may be passed to functions running in different goroutines; +// Contexts are safe for simultaneous use by multiple goroutines. +// +// See http://blog.golang.org/context for example code for a server that uses +// Contexts. +package context // import "golang.org/x/net/context" + +import ( + "errors" + "fmt" + "sync" + "time" +) + +// A Context carries a deadline, a cancelation signal, and other values across +// API boundaries. +// +// Context's methods may be called by multiple goroutines simultaneously. +type Context interface { + // Deadline returns the time when work done on behalf of this context + // should be canceled. Deadline returns ok==false when no deadline is + // set. Successive calls to Deadline return the same results. + Deadline() (deadline time.Time, ok bool) + + // Done returns a channel that's closed when work done on behalf of this + // context should be canceled. Done may return nil if this context can + // never be canceled. Successive calls to Done return the same value. + // + // WithCancel arranges for Done to be closed when cancel is called; + // WithDeadline arranges for Done to be closed when the deadline + // expires; WithTimeout arranges for Done to be closed when the timeout + // elapses. + // + // Done is provided for use in select statements: + // + // // Stream generates values with DoSomething and sends them to out + // // until DoSomething returns an error or ctx.Done is closed. + // func Stream(ctx context.Context, out <-chan Value) error { + // for { + // v, err := DoSomething(ctx) + // if err != nil { + // return err + // } + // select { + // case <-ctx.Done(): + // return ctx.Err() + // case out <- v: + // } + // } + // } + // + // See http://blog.golang.org/pipelines for more examples of how to use + // a Done channel for cancelation. + Done() <-chan struct{} + + // Err returns a non-nil error value after Done is closed. Err returns + // Canceled if the context was canceled or DeadlineExceeded if the + // context's deadline passed. No other values for Err are defined. + // After Done is closed, successive calls to Err return the same value. + Err() error + + // Value returns the value associated with this context for key, or nil + // if no value is associated with key. Successive calls to Value with + // the same key returns the same result. + // + // Use context values only for request-scoped data that transits + // processes and API boundaries, not for passing optional parameters to + // functions. + // + // A key identifies a specific value in a Context. Functions that wish + // to store values in Context typically allocate a key in a global + // variable then use that key as the argument to context.WithValue and + // Context.Value. A key can be any type that supports equality; + // packages should define keys as an unexported type to avoid + // collisions. + // + // Packages that define a Context key should provide type-safe accessors + // for the values stores using that key: + // + // // Package user defines a User type that's stored in Contexts. + // package user + // + // import "golang.org/x/net/context" + // + // // User is the type of value stored in the Contexts. + // type User struct {...} + // + // // key is an unexported type for keys defined in this package. + // // This prevents collisions with keys defined in other packages. + // type key int + // + // // userKey is the key for user.User values in Contexts. It is + // // unexported; clients use user.NewContext and user.FromContext + // // instead of using this key directly. + // var userKey key = 0 + // + // // NewContext returns a new Context that carries value u. + // func NewContext(ctx context.Context, u *User) context.Context { + // return context.WithValue(ctx, userKey, u) + // } + // + // // FromContext returns the User value stored in ctx, if any. + // func FromContext(ctx context.Context) (*User, bool) { + // u, ok := ctx.Value(userKey).(*User) + // return u, ok + // } + Value(key interface{}) interface{} +} + +// Canceled is the error returned by Context.Err when the context is canceled. +var Canceled = errors.New("context canceled") + +// DeadlineExceeded is the error returned by Context.Err when the context's +// deadline passes. +var DeadlineExceeded = errors.New("context deadline exceeded") + +// An emptyCtx is never canceled, has no values, and has no deadline. It is not +// struct{}, since vars of this type must have distinct addresses. +type emptyCtx int + +func (*emptyCtx) Deadline() (deadline time.Time, ok bool) { + return +} + +func (*emptyCtx) Done() <-chan struct{} { + return nil +} + +func (*emptyCtx) Err() error { + return nil +} + +func (*emptyCtx) Value(key interface{}) interface{} { + return nil +} + +func (e *emptyCtx) String() string { + switch e { + case background: + return "context.Background" + case todo: + return "context.TODO" + } + return "unknown empty Context" +} + +var ( + background = new(emptyCtx) + todo = new(emptyCtx) +) + +// Background returns a non-nil, empty Context. It is never canceled, has no +// values, and has no deadline. It is typically used by the main function, +// initialization, and tests, and as the top-level Context for incoming +// requests. +func Background() Context { + return background +} + +// TODO returns a non-nil, empty Context. Code should use context.TODO when +// it's unclear which Context to use or it's is not yet available (because the +// surrounding function has not yet been extended to accept a Context +// parameter). TODO is recognized by static analysis tools that determine +// whether Contexts are propagated correctly in a program. +func TODO() Context { + return todo +} + +// A CancelFunc tells an operation to abandon its work. +// A CancelFunc does not wait for the work to stop. +// After the first call, subsequent calls to a CancelFunc do nothing. +type CancelFunc func() + +// WithCancel returns a copy of parent with a new Done channel. The returned +// context's Done channel is closed when the returned cancel function is called +// or when the parent context's Done channel is closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { + c := newCancelCtx(parent) + propagateCancel(parent, &c) + return &c, func() { c.cancel(true, Canceled) } +} + +// newCancelCtx returns an initialized cancelCtx. +func newCancelCtx(parent Context) cancelCtx { + return cancelCtx{ + Context: parent, + done: make(chan struct{}), + } +} + +// propagateCancel arranges for child to be canceled when parent is. +func propagateCancel(parent Context, child canceler) { + if parent.Done() == nil { + return // parent is never canceled + } + if p, ok := parentCancelCtx(parent); ok { + p.mu.Lock() + if p.err != nil { + // parent has already been canceled + child.cancel(false, p.err) + } else { + if p.children == nil { + p.children = make(map[canceler]bool) + } + p.children[child] = true + } + p.mu.Unlock() + } else { + go func() { + select { + case <-parent.Done(): + child.cancel(false, parent.Err()) + case <-child.Done(): + } + }() + } +} + +// parentCancelCtx follows a chain of parent references until it finds a +// *cancelCtx. This function understands how each of the concrete types in this +// package represents its parent. +func parentCancelCtx(parent Context) (*cancelCtx, bool) { + for { + switch c := parent.(type) { + case *cancelCtx: + return c, true + case *timerCtx: + return &c.cancelCtx, true + case *valueCtx: + parent = c.Context + default: + return nil, false + } + } +} + +// removeChild removes a context from its parent. +func removeChild(parent Context, child canceler) { + p, ok := parentCancelCtx(parent) + if !ok { + return + } + p.mu.Lock() + if p.children != nil { + delete(p.children, child) + } + p.mu.Unlock() +} + +// A canceler is a context type that can be canceled directly. The +// implementations are *cancelCtx and *timerCtx. +type canceler interface { + cancel(removeFromParent bool, err error) + Done() <-chan struct{} +} + +// A cancelCtx can be canceled. When canceled, it also cancels any children +// that implement canceler. +type cancelCtx struct { + Context + + done chan struct{} // closed by the first cancel call. + + mu sync.Mutex + children map[canceler]bool // set to nil by the first cancel call + err error // set to non-nil by the first cancel call +} + +func (c *cancelCtx) Done() <-chan struct{} { + return c.done +} + +func (c *cancelCtx) Err() error { + c.mu.Lock() + defer c.mu.Unlock() + return c.err +} + +func (c *cancelCtx) String() string { + return fmt.Sprintf("%v.WithCancel", c.Context) +} + +// cancel closes c.done, cancels each of c's children, and, if +// removeFromParent is true, removes c from its parent's children. +func (c *cancelCtx) cancel(removeFromParent bool, err error) { + if err == nil { + panic("context: internal error: missing cancel error") + } + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return // already canceled + } + c.err = err + close(c.done) + for child := range c.children { + // NOTE: acquiring the child's lock while holding parent's lock. + child.cancel(false, err) + } + c.children = nil + c.mu.Unlock() + + if removeFromParent { + removeChild(c.Context, c) + } +} + +// WithDeadline returns a copy of the parent context with the deadline adjusted +// to be no later than d. If the parent's deadline is already earlier than d, +// WithDeadline(parent, d) is semantically equivalent to parent. The returned +// context's Done channel is closed when the deadline expires, when the returned +// cancel function is called, or when the parent context's Done channel is +// closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { + if cur, ok := parent.Deadline(); ok && cur.Before(deadline) { + // The current deadline is already sooner than the new one. + return WithCancel(parent) + } + c := &timerCtx{ + cancelCtx: newCancelCtx(parent), + deadline: deadline, + } + propagateCancel(parent, c) + d := deadline.Sub(time.Now()) + if d <= 0 { + c.cancel(true, DeadlineExceeded) // deadline has already passed + return c, func() { c.cancel(true, Canceled) } + } + c.mu.Lock() + defer c.mu.Unlock() + if c.err == nil { + c.timer = time.AfterFunc(d, func() { + c.cancel(true, DeadlineExceeded) + }) + } + return c, func() { c.cancel(true, Canceled) } +} + +// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to +// implement Done and Err. It implements cancel by stopping its timer then +// delegating to cancelCtx.cancel. +type timerCtx struct { + cancelCtx + timer *time.Timer // Under cancelCtx.mu. + + deadline time.Time +} + +func (c *timerCtx) Deadline() (deadline time.Time, ok bool) { + return c.deadline, true +} + +func (c *timerCtx) String() string { + return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now())) +} + +func (c *timerCtx) cancel(removeFromParent bool, err error) { + c.cancelCtx.cancel(false, err) + if removeFromParent { + // Remove this timerCtx from its parent cancelCtx's children. + removeChild(c.cancelCtx.Context, c) + } + c.mu.Lock() + if c.timer != nil { + c.timer.Stop() + c.timer = nil + } + c.mu.Unlock() +} + +// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete: +// +// func slowOperationWithTimeout(ctx context.Context) (Result, error) { +// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) +// defer cancel() // releases resources if slowOperation completes before timeout elapses +// return slowOperation(ctx) +// } +func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { + return WithDeadline(parent, time.Now().Add(timeout)) +} + +// WithValue returns a copy of parent in which the value associated with key is +// val. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +func WithValue(parent Context, key interface{}, val interface{}) Context { + return &valueCtx{parent, key, val} +} + +// A valueCtx carries a key-value pair. It implements Value for that key and +// delegates all other calls to the embedded Context. +type valueCtx struct { + Context + key, val interface{} +} + +func (c *valueCtx) String() string { + return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val) +} + +func (c *valueCtx) Value(key interface{}) interface{} { + if c.key == key { + return c.val + } + return c.Context.Value(key) +} diff --git a/Godeps/_workspace/src/golang.org/x/net/context/context_test.go b/Godeps/_workspace/src/golang.org/x/net/context/context_test.go new file mode 100644 index 0000000000..faf67722a0 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/context/context_test.go @@ -0,0 +1,575 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package context + +import ( + "fmt" + "math/rand" + "runtime" + "strings" + "sync" + "testing" + "time" +) + +// otherContext is a Context that's not one of the types defined in context.go. +// This lets us test code paths that differ based on the underlying type of the +// Context. +type otherContext struct { + Context +} + +func TestBackground(t *testing.T) { + c := Background() + if c == nil { + t.Fatalf("Background returned nil") + } + select { + case x := <-c.Done(): + t.Errorf("<-c.Done() == %v want nothing (it should block)", x) + default: + } + if got, want := fmt.Sprint(c), "context.Background"; got != want { + t.Errorf("Background().String() = %q want %q", got, want) + } +} + +func TestTODO(t *testing.T) { + c := TODO() + if c == nil { + t.Fatalf("TODO returned nil") + } + select { + case x := <-c.Done(): + t.Errorf("<-c.Done() == %v want nothing (it should block)", x) + default: + } + if got, want := fmt.Sprint(c), "context.TODO"; got != want { + t.Errorf("TODO().String() = %q want %q", got, want) + } +} + +func TestWithCancel(t *testing.T) { + c1, cancel := WithCancel(Background()) + + if got, want := fmt.Sprint(c1), "context.Background.WithCancel"; got != want { + t.Errorf("c1.String() = %q want %q", got, want) + } + + o := otherContext{c1} + c2, _ := WithCancel(o) + contexts := []Context{c1, o, c2} + + for i, c := range contexts { + if d := c.Done(); d == nil { + t.Errorf("c[%d].Done() == %v want non-nil", i, d) + } + if e := c.Err(); e != nil { + t.Errorf("c[%d].Err() == %v want nil", i, e) + } + + select { + case x := <-c.Done(): + t.Errorf("<-c.Done() == %v want nothing (it should block)", x) + default: + } + } + + cancel() + time.Sleep(100 * time.Millisecond) // let cancelation propagate + + for i, c := range contexts { + select { + case <-c.Done(): + default: + t.Errorf("<-c[%d].Done() blocked, but shouldn't have", i) + } + if e := c.Err(); e != Canceled { + t.Errorf("c[%d].Err() == %v want %v", i, e, Canceled) + } + } +} + +func TestParentFinishesChild(t *testing.T) { + // Context tree: + // parent -> cancelChild + // parent -> valueChild -> timerChild + parent, cancel := WithCancel(Background()) + cancelChild, stop := WithCancel(parent) + defer stop() + valueChild := WithValue(parent, "key", "value") + timerChild, stop := WithTimeout(valueChild, 10000*time.Hour) + defer stop() + + select { + case x := <-parent.Done(): + t.Errorf("<-parent.Done() == %v want nothing (it should block)", x) + case x := <-cancelChild.Done(): + t.Errorf("<-cancelChild.Done() == %v want nothing (it should block)", x) + case x := <-timerChild.Done(): + t.Errorf("<-timerChild.Done() == %v want nothing (it should block)", x) + case x := <-valueChild.Done(): + t.Errorf("<-valueChild.Done() == %v want nothing (it should block)", x) + default: + } + + // The parent's children should contain the two cancelable children. + pc := parent.(*cancelCtx) + cc := cancelChild.(*cancelCtx) + tc := timerChild.(*timerCtx) + pc.mu.Lock() + if len(pc.children) != 2 || !pc.children[cc] || !pc.children[tc] { + t.Errorf("bad linkage: pc.children = %v, want %v and %v", + pc.children, cc, tc) + } + pc.mu.Unlock() + + if p, ok := parentCancelCtx(cc.Context); !ok || p != pc { + t.Errorf("bad linkage: parentCancelCtx(cancelChild.Context) = %v, %v want %v, true", p, ok, pc) + } + if p, ok := parentCancelCtx(tc.Context); !ok || p != pc { + t.Errorf("bad linkage: parentCancelCtx(timerChild.Context) = %v, %v want %v, true", p, ok, pc) + } + + cancel() + + pc.mu.Lock() + if len(pc.children) != 0 { + t.Errorf("pc.cancel didn't clear pc.children = %v", pc.children) + } + pc.mu.Unlock() + + // parent and children should all be finished. + check := func(ctx Context, name string) { + select { + case <-ctx.Done(): + default: + t.Errorf("<-%s.Done() blocked, but shouldn't have", name) + } + if e := ctx.Err(); e != Canceled { + t.Errorf("%s.Err() == %v want %v", name, e, Canceled) + } + } + check(parent, "parent") + check(cancelChild, "cancelChild") + check(valueChild, "valueChild") + check(timerChild, "timerChild") + + // WithCancel should return a canceled context on a canceled parent. + precanceledChild := WithValue(parent, "key", "value") + select { + case <-precanceledChild.Done(): + default: + t.Errorf("<-precanceledChild.Done() blocked, but shouldn't have") + } + if e := precanceledChild.Err(); e != Canceled { + t.Errorf("precanceledChild.Err() == %v want %v", e, Canceled) + } +} + +func TestChildFinishesFirst(t *testing.T) { + cancelable, stop := WithCancel(Background()) + defer stop() + for _, parent := range []Context{Background(), cancelable} { + child, cancel := WithCancel(parent) + + select { + case x := <-parent.Done(): + t.Errorf("<-parent.Done() == %v want nothing (it should block)", x) + case x := <-child.Done(): + t.Errorf("<-child.Done() == %v want nothing (it should block)", x) + default: + } + + cc := child.(*cancelCtx) + pc, pcok := parent.(*cancelCtx) // pcok == false when parent == Background() + if p, ok := parentCancelCtx(cc.Context); ok != pcok || (ok && pc != p) { + t.Errorf("bad linkage: parentCancelCtx(cc.Context) = %v, %v want %v, %v", p, ok, pc, pcok) + } + + if pcok { + pc.mu.Lock() + if len(pc.children) != 1 || !pc.children[cc] { + t.Errorf("bad linkage: pc.children = %v, cc = %v", pc.children, cc) + } + pc.mu.Unlock() + } + + cancel() + + if pcok { + pc.mu.Lock() + if len(pc.children) != 0 { + t.Errorf("child's cancel didn't remove self from pc.children = %v", pc.children) + } + pc.mu.Unlock() + } + + // child should be finished. + select { + case <-child.Done(): + default: + t.Errorf("<-child.Done() blocked, but shouldn't have") + } + if e := child.Err(); e != Canceled { + t.Errorf("child.Err() == %v want %v", e, Canceled) + } + + // parent should not be finished. + select { + case x := <-parent.Done(): + t.Errorf("<-parent.Done() == %v want nothing (it should block)", x) + default: + } + if e := parent.Err(); e != nil { + t.Errorf("parent.Err() == %v want nil", e) + } + } +} + +func testDeadline(c Context, wait time.Duration, t *testing.T) { + select { + case <-time.After(wait): + t.Fatalf("context should have timed out") + case <-c.Done(): + } + if e := c.Err(); e != DeadlineExceeded { + t.Errorf("c.Err() == %v want %v", e, DeadlineExceeded) + } +} + +func TestDeadline(t *testing.T) { + c, _ := WithDeadline(Background(), time.Now().Add(100*time.Millisecond)) + if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) { + t.Errorf("c.String() = %q want prefix %q", got, prefix) + } + testDeadline(c, 200*time.Millisecond, t) + + c, _ = WithDeadline(Background(), time.Now().Add(100*time.Millisecond)) + o := otherContext{c} + testDeadline(o, 200*time.Millisecond, t) + + c, _ = WithDeadline(Background(), time.Now().Add(100*time.Millisecond)) + o = otherContext{c} + c, _ = WithDeadline(o, time.Now().Add(300*time.Millisecond)) + testDeadline(c, 200*time.Millisecond, t) +} + +func TestTimeout(t *testing.T) { + c, _ := WithTimeout(Background(), 100*time.Millisecond) + if got, prefix := fmt.Sprint(c), "context.Background.WithDeadline("; !strings.HasPrefix(got, prefix) { + t.Errorf("c.String() = %q want prefix %q", got, prefix) + } + testDeadline(c, 200*time.Millisecond, t) + + c, _ = WithTimeout(Background(), 100*time.Millisecond) + o := otherContext{c} + testDeadline(o, 200*time.Millisecond, t) + + c, _ = WithTimeout(Background(), 100*time.Millisecond) + o = otherContext{c} + c, _ = WithTimeout(o, 300*time.Millisecond) + testDeadline(c, 200*time.Millisecond, t) +} + +func TestCanceledTimeout(t *testing.T) { + c, _ := WithTimeout(Background(), 200*time.Millisecond) + o := otherContext{c} + c, cancel := WithTimeout(o, 400*time.Millisecond) + cancel() + time.Sleep(100 * time.Millisecond) // let cancelation propagate + select { + case <-c.Done(): + default: + t.Errorf("<-c.Done() blocked, but shouldn't have") + } + if e := c.Err(); e != Canceled { + t.Errorf("c.Err() == %v want %v", e, Canceled) + } +} + +type key1 int +type key2 int + +var k1 = key1(1) +var k2 = key2(1) // same int as k1, different type +var k3 = key2(3) // same type as k2, different int + +func TestValues(t *testing.T) { + check := func(c Context, nm, v1, v2, v3 string) { + if v, ok := c.Value(k1).(string); ok == (len(v1) == 0) || v != v1 { + t.Errorf(`%s.Value(k1).(string) = %q, %t want %q, %t`, nm, v, ok, v1, len(v1) != 0) + } + if v, ok := c.Value(k2).(string); ok == (len(v2) == 0) || v != v2 { + t.Errorf(`%s.Value(k2).(string) = %q, %t want %q, %t`, nm, v, ok, v2, len(v2) != 0) + } + if v, ok := c.Value(k3).(string); ok == (len(v3) == 0) || v != v3 { + t.Errorf(`%s.Value(k3).(string) = %q, %t want %q, %t`, nm, v, ok, v3, len(v3) != 0) + } + } + + c0 := Background() + check(c0, "c0", "", "", "") + + c1 := WithValue(Background(), k1, "c1k1") + check(c1, "c1", "c1k1", "", "") + + if got, want := fmt.Sprint(c1), `context.Background.WithValue(1, "c1k1")`; got != want { + t.Errorf("c.String() = %q want %q", got, want) + } + + c2 := WithValue(c1, k2, "c2k2") + check(c2, "c2", "c1k1", "c2k2", "") + + c3 := WithValue(c2, k3, "c3k3") + check(c3, "c2", "c1k1", "c2k2", "c3k3") + + c4 := WithValue(c3, k1, nil) + check(c4, "c4", "", "c2k2", "c3k3") + + o0 := otherContext{Background()} + check(o0, "o0", "", "", "") + + o1 := otherContext{WithValue(Background(), k1, "c1k1")} + check(o1, "o1", "c1k1", "", "") + + o2 := WithValue(o1, k2, "o2k2") + check(o2, "o2", "c1k1", "o2k2", "") + + o3 := otherContext{c4} + check(o3, "o3", "", "c2k2", "c3k3") + + o4 := WithValue(o3, k3, nil) + check(o4, "o4", "", "c2k2", "") +} + +func TestAllocs(t *testing.T) { + bg := Background() + for _, test := range []struct { + desc string + f func() + limit float64 + gccgoLimit float64 + }{ + { + desc: "Background()", + f: func() { Background() }, + limit: 0, + gccgoLimit: 0, + }, + { + desc: fmt.Sprintf("WithValue(bg, %v, nil)", k1), + f: func() { + c := WithValue(bg, k1, nil) + c.Value(k1) + }, + limit: 3, + gccgoLimit: 3, + }, + { + desc: "WithTimeout(bg, 15*time.Millisecond)", + f: func() { + c, _ := WithTimeout(bg, 15*time.Millisecond) + <-c.Done() + }, + limit: 8, + gccgoLimit: 13, + }, + { + desc: "WithCancel(bg)", + f: func() { + c, cancel := WithCancel(bg) + cancel() + <-c.Done() + }, + limit: 5, + gccgoLimit: 8, + }, + { + desc: "WithTimeout(bg, 100*time.Millisecond)", + f: func() { + c, cancel := WithTimeout(bg, 100*time.Millisecond) + cancel() + <-c.Done() + }, + limit: 8, + gccgoLimit: 25, + }, + } { + limit := test.limit + if runtime.Compiler == "gccgo" { + // gccgo does not yet do escape analysis. + // TOOD(iant): Remove this when gccgo does do escape analysis. + limit = test.gccgoLimit + } + if n := testing.AllocsPerRun(100, test.f); n > limit { + t.Errorf("%s allocs = %f want %d", test.desc, n, int(limit)) + } + } +} + +func TestSimultaneousCancels(t *testing.T) { + root, cancel := WithCancel(Background()) + m := map[Context]CancelFunc{root: cancel} + q := []Context{root} + // Create a tree of contexts. + for len(q) != 0 && len(m) < 100 { + parent := q[0] + q = q[1:] + for i := 0; i < 4; i++ { + ctx, cancel := WithCancel(parent) + m[ctx] = cancel + q = append(q, ctx) + } + } + // Start all the cancels in a random order. + var wg sync.WaitGroup + wg.Add(len(m)) + for _, cancel := range m { + go func(cancel CancelFunc) { + cancel() + wg.Done() + }(cancel) + } + // Wait on all the contexts in a random order. + for ctx := range m { + select { + case <-ctx.Done(): + case <-time.After(1 * time.Second): + buf := make([]byte, 10<<10) + n := runtime.Stack(buf, true) + t.Fatalf("timed out waiting for <-ctx.Done(); stacks:\n%s", buf[:n]) + } + } + // Wait for all the cancel functions to return. + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(1 * time.Second): + buf := make([]byte, 10<<10) + n := runtime.Stack(buf, true) + t.Fatalf("timed out waiting for cancel functions; stacks:\n%s", buf[:n]) + } +} + +func TestInterlockedCancels(t *testing.T) { + parent, cancelParent := WithCancel(Background()) + child, cancelChild := WithCancel(parent) + go func() { + parent.Done() + cancelChild() + }() + cancelParent() + select { + case <-child.Done(): + case <-time.After(1 * time.Second): + buf := make([]byte, 10<<10) + n := runtime.Stack(buf, true) + t.Fatalf("timed out waiting for child.Done(); stacks:\n%s", buf[:n]) + } +} + +func TestLayersCancel(t *testing.T) { + testLayers(t, time.Now().UnixNano(), false) +} + +func TestLayersTimeout(t *testing.T) { + testLayers(t, time.Now().UnixNano(), true) +} + +func testLayers(t *testing.T, seed int64, testTimeout bool) { + rand.Seed(seed) + errorf := func(format string, a ...interface{}) { + t.Errorf(fmt.Sprintf("seed=%d: %s", seed, format), a...) + } + const ( + timeout = 200 * time.Millisecond + minLayers = 30 + ) + type value int + var ( + vals []*value + cancels []CancelFunc + numTimers int + ctx = Background() + ) + for i := 0; i < minLayers || numTimers == 0 || len(cancels) == 0 || len(vals) == 0; i++ { + switch rand.Intn(3) { + case 0: + v := new(value) + ctx = WithValue(ctx, v, v) + vals = append(vals, v) + case 1: + var cancel CancelFunc + ctx, cancel = WithCancel(ctx) + cancels = append(cancels, cancel) + case 2: + var cancel CancelFunc + ctx, cancel = WithTimeout(ctx, timeout) + cancels = append(cancels, cancel) + numTimers++ + } + } + checkValues := func(when string) { + for _, key := range vals { + if val := ctx.Value(key).(*value); key != val { + errorf("%s: ctx.Value(%p) = %p want %p", when, key, val, key) + } + } + } + select { + case <-ctx.Done(): + errorf("ctx should not be canceled yet") + default: + } + if s, prefix := fmt.Sprint(ctx), "context.Background."; !strings.HasPrefix(s, prefix) { + t.Errorf("ctx.String() = %q want prefix %q", s, prefix) + } + t.Log(ctx) + checkValues("before cancel") + if testTimeout { + select { + case <-ctx.Done(): + case <-time.After(timeout + timeout/10): + errorf("ctx should have timed out") + } + checkValues("after timeout") + } else { + cancel := cancels[rand.Intn(len(cancels))] + cancel() + select { + case <-ctx.Done(): + default: + errorf("ctx should be canceled") + } + checkValues("after cancel") + } +} + +func TestCancelRemoves(t *testing.T) { + checkChildren := func(when string, ctx Context, want int) { + if got := len(ctx.(*cancelCtx).children); got != want { + t.Errorf("%s: context has %d children, want %d", when, got, want) + } + } + + ctx, _ := WithCancel(Background()) + checkChildren("after creation", ctx, 0) + _, cancel := WithCancel(ctx) + checkChildren("with WithCancel child ", ctx, 1) + cancel() + checkChildren("after cancelling WithCancel child", ctx, 0) + + ctx, _ = WithCancel(Background()) + checkChildren("after creation", ctx, 0) + _, cancel = WithTimeout(ctx, 60*time.Minute) + checkChildren("with WithTimeout child ", ctx, 1) + cancel() + checkChildren("after cancelling WithTimeout child", ctx, 0) +} diff --git a/Godeps/_workspace/src/golang.org/x/net/context/withtimeout_test.go b/Godeps/_workspace/src/golang.org/x/net/context/withtimeout_test.go new file mode 100644 index 0000000000..a6754dc368 --- /dev/null +++ b/Godeps/_workspace/src/golang.org/x/net/context/withtimeout_test.go @@ -0,0 +1,26 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package context_test + +import ( + "fmt" + "time" + + "golang.org/x/net/context" +) + +func ExampleWithTimeout() { + // Pass a context with a timeout to tell a blocking function that it + // should abandon its work after the timeout elapses. + ctx, _ := context.WithTimeout(context.Background(), 100*time.Millisecond) + select { + case <-time.After(200 * time.Millisecond): + fmt.Println("overslept") + case <-ctx.Done(): + fmt.Println(ctx.Err()) // prints "context deadline exceeded" + } + // Output: + // context deadline exceeded +} diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 64044c4214..2df3a4e8da 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -113,7 +114,7 @@ func run(ctx *cli.Context) { glog.SetV(ctx.GlobalInt(VerbosityFlag.Name)) db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, access.NewDbChainAccess(db)) sender := statedb.CreateAccount(common.StringToAddress("sender")) receiver := statedb.CreateAccount(common.StringToAddress("receiver")) receiver.SetCode(common.Hex2Bytes(ctx.GlobalString(CodeFlag.Name))) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 80f3777d6d..ef5680960f 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -179,7 +180,7 @@ func dump(ctx *cli.Context) { fmt.Println("{}") utils.Fatalf("block not found") } else { - state, err := state.New(block.Root(), chainDb) + state, err := state.New(block.Root(), access.NewDbChainAccess(chainDb)) if err != nil { utils.Fatalf("could not create new state: %v", err) return diff --git a/cmd/geth/main.go b/cmd/geth/main.go index d63d20580a..a8434626f8 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" @@ -555,12 +556,13 @@ func blockRecovery(ctx *cli.Context) { if err != nil { glog.Fatalln("could not open db:", err) } + ca := access.NewDbChainAccess(blockDb) var block *types.Block if arg[0] == '#' { - block = core.GetBlock(blockDb, core.GetCanonicalHash(blockDb, common.String2Big(arg[1:]).Uint64())) + block = core.GetBlock(ca, core.GetCanonicalHash(blockDb, common.String2Big(arg[1:]).Uint64())) } else { - block = core.GetBlock(blockDb, common.HexToHash(arg)) + block = core.GetBlock(ca, common.HexToHash(arg)) } if block == nil { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8335517dff..22212df466 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" @@ -552,12 +553,12 @@ func MakeChain(ctx *cli.Context) (chain *core.BlockChain, chainDb ethdb.Database eventMux := new(event.TypeMux) pow := ethash.New() //genesis := core.GenesisBlock(uint64(ctx.GlobalInt(GenesisNonceFlag.Name)), blockDB) - chain, err = core.NewBlockChain(chainDb, pow, eventMux) + chain, err = core.NewBlockChain(access.NewDbChainAccess(chainDb), pow, eventMux) if err != nil { Fatalf("Could not start chainmanager: %v", err) } - proc := core.NewBlockProcessor(chainDb, pow, chain, eventMux) + proc := core.NewBlockProcessor(access.NewDbChainAccess(chainDb), pow, chain, eventMux) chain.SetProcessor(proc) return chain, chainDb } @@ -665,4 +666,4 @@ func ParamToAddress(addr string, am *accounts.Manager) (addrHex string, err erro addrHex = addr } return -} +} \ No newline at end of file diff --git a/core/access/chain_access.go b/core/access/chain_access.go new file mode 100644 index 0000000000..515fc6f85b --- /dev/null +++ b/core/access/chain_access.go @@ -0,0 +1,219 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package access provides a layer to handle local blockchain database and +// on-demand network retrieval +package access + +import ( + "errors" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "golang.org/x/net/context" +) + +var ( + errNotInDb = errors.New("object not found in database") +) + +const LogLevel = logger.Debug + +var ( + requestTimeout = time.Millisecond * 300 + retryPeers = time.Second * 1 +) + +// ChainAccess provides access to blockchain and state data through local +// database and optionally also on-demand network retrieval +type ChainAccess struct { + db ethdb.Database + odr bool // light client mode, odr enabled + lock sync.Mutex + sentReqs map[uint64]*sentReq + sentReqCnt uint64 + peers *peerSet +} + +// requestFunc is a function that requests some data from a peer +type requestFunc func(*Peer) error + +// validatorFunc is a function that processes a message and returns true if +// it was a meaningful answer to a given request +type validatorFunc func(*Msg) bool + +// sentReq is a request waiting for an answer that satisfies its valFunc +type sentReq struct { + valFunc validatorFunc + deliverChan chan *Msg +} + +// NewDbChainAccess creates a ChainAccess with ODR disabled +func NewDbChainAccess(db ethdb.Database) *ChainAccess { + return NewChainAccess(db, false) +} + +// NewChainAccess create a ChainAccess with optional ODR +func NewChainAccess(db ethdb.Database, odr bool) *ChainAccess { + return &ChainAccess{ + db: db, + peers: newPeerSet(), + sentReqs: make(map[uint64]*sentReq), + odr: odr, + } +} + +// Db returns the local database assigned to the ChainAccess object +func (self *ChainAccess) Db() ethdb.Database { + return self.db +} + +// OdrEnabled returns true if this ChainAccess is capable of doing ODR requests +func (self *ChainAccess) OdrEnabled() bool { + return self.odr +} + +// RegisterPeer registers a new LES peer to the ODR capable peer set +func (self *ChainAccess) RegisterPeer(id string, version int, head common.Hash, getBlockBodies getBlockBodiesFn, getNodeData getNodeDataFn, getReceipts getReceiptsFn, getProofs getProofsFn) error { + glog.V(logger.Detail).Infoln("Registering peer", id) + if err := self.peers.Register(newPeer(id, version, head, getBlockBodies, getNodeData, getReceipts, getProofs)); err != nil { + glog.V(logger.Error).Infoln("Register failed:", err) + return err + } + return nil +} + +// UnregisterPeer removes a peer from the ODR capable peer set +func (self *ChainAccess) UnregisterPeer(id string) { + self.peers.Unregister(id) +} + +const ( + MsgBlockBodies = iota + MsgNodeData + MsgReceipts + MsgProofs +) + +// Msg encodes a LES message that delivers reply data for a request +type Msg struct { + MsgType int + Obj interface{} +} + +// ObjectAccess is the ODR request interface (passed to Retrieve, functions called by Retrieve and Deliver) +// DbGet() tries to retrieve the object from the local database (object is stored by the request struct in memory if retrieved) +// DbPut() stores it in the local database +// Request(*Peer) requests it from a LES peer +// Valid(*Msg) checks if a message is a valid answer to this request and stores the retrieved object in memory +type ObjectAccess interface { + // database storage + DbGet() bool + DbPut() + // network retrieval + Request(*Peer) error + Valid(*Msg) bool // if true, keeps the retrieved object +} + +// Deliver is called by the LES protocol manager to deliver ODR reply messages to waiting requests +func (self *ChainAccess) Deliver(id string, msg *Msg) (processed bool) { + self.lock.Lock() + defer self.lock.Unlock() + + for i, req := range self.sentReqs { + if req.valFunc(msg) { + req.deliverChan <- msg + delete(self.sentReqs, i) + return true + } + } + return false +} + +// networkRequest sends a request to known peers until an answer is received +// or the context is cancelled +func (self *ChainAccess) networkRequest(ctx context.Context, rqFunc requestFunc, valFunc validatorFunc) (*Msg, error) { + req := &sentReq{ + deliverChan: make(chan *Msg), + valFunc: valFunc, + } + self.lock.Lock() + reqCnt := self.sentReqCnt + self.sentReqCnt++ + self.sentReqs[reqCnt] = req + self.lock.Unlock() + + defer func() { + self.lock.Lock() + delete(self.sentReqs, reqCnt) + self.lock.Unlock() + }() + + var msg *Msg + + for { + peers := self.peers.BestPeers() + if len(peers) == 0 { + select { + case <-ctx.Done(): + setTerminated(ctx) + return nil, ctx.Err() + case <-time.After(retryPeers): + } + } + for _, peer := range peers { + rqFunc(peer) + select { + case <-ctx.Done(): + setTerminated(ctx) + return nil, ctx.Err() + case msg = <-req.deliverChan: + peer.Promote() + glog.V(LogLevel).Infof("networkRequest success") + return msg, nil + case <-time.After(requestTimeout): + peer.Demote() + glog.V(LogLevel).Infof("networkRequest timeout") + } + } + } +} + +// Retrieve tries to fetch an object from the local db, then from the LES network. +// If the network retrieval was successful, it stores the object in local db. +func (self *ChainAccess) Retrieve(ctx context.Context, obj ObjectAccess) (err error) { + // look in db + if obj.DbGet() { + return nil + } + if IsOdrContext(ctx) { + // not found in db, trying the network + _, err = self.networkRequest(ctx, obj.Request, obj.Valid) + if err == nil { + // retrieved from network, store in db + obj.DbPut() + } else { + glog.V(LogLevel).Infof("networkRequest err = %v", err) + } + return + } else { + return errNotInDb + } +} diff --git a/core/access/context.go b/core/access/context.go new file mode 100644 index 0000000000..9d68e129f6 --- /dev/null +++ b/core/access/context.go @@ -0,0 +1,76 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package access provides a layer to handle local blockchain database and +// on-demand network retrieval +package access + +import ( + "sync/atomic" + "time" + + "golang.org/x/net/context" +) + +// NoOdr is the default context when ODR is not used +var NoOdr = context.Background() + +// NewContext creates an ODR context, carrying channel ID and a "terminated" flag +func NewContext(id *OdrChannelID) (context.Context, context.CancelFunc) { + ctx := context.Background() + ctx = context.WithValue(ctx, 0, id) + ctx = context.WithValue(ctx, 1, new(int32)) + return context.WithTimeout(ctx, id.timeout) +} + +// IsOdrContext returns true if ctx is an ODR-enabled context +func IsOdrContext(ctx context.Context) bool { + _, ok := ctx.Value(0).(*OdrChannelID) + return ok +} + +// setTerminated switches the "terminated" flag on, notifying caller that any +// result it received should be considered invalid +func setTerminated(ctx context.Context) { + ptr, ok := ctx.Value(1).(*int32) + if ok { + atomic.StoreInt32(ptr, 1) + } +} + +// Terminated returns true if the "terminated" flag was switched on, meaning +// that any result received should be considered invalid +func Terminated(ctx context.Context) bool { + ptr, ok := ctx.Value(1).(*int32) + if ok { + return atomic.LoadInt32(ptr) == 1 + } else { + return false + } +} + +// OdrChannelID is a permanent identifier of a source from where +// requests can come (like an RPC channel). +// (needed for future functions like "list of my waiting requests" and +// "cancel all requests from this channel") +type OdrChannelID struct { + timeout time.Duration +} + +// NewChannelID creates a new OdrChannelID with a channel-specific timeout parameter +func NewChannelID(timeout time.Duration) *OdrChannelID { + return &OdrChannelID{timeout: timeout} +} diff --git a/core/access/peer.go b/core/access/peer.go new file mode 100644 index 0000000000..2d6e979291 --- /dev/null +++ b/core/access/peer.go @@ -0,0 +1,181 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package access provides a layer to handle local blockchain database and +// on-demand network retrieval +package access + +import ( + "errors" + "sync" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" +) + +var ( + errAlreadyRegistered = errors.New("peer is already registered") + errNotRegistered = errors.New("peer is not registered") + errNoOdr = errors.New("peer cannot serve on-demand requests") +) + +type ProofReq struct { + Root common.Hash + Key []byte +} + +type getBlockBodiesFn func([]common.Hash) error +type getNodeDataFn func([]common.Hash) error +type getReceiptsFn func([]common.Hash) error +type getProofsFn func([]*ProofReq) error + +// Peer stores ODR-specific information about LES peers that are able to serve requests +type Peer struct { + id string // Unique identifier of the peer + head common.Hash // Hash of the peers latest known block + + rep int32 // Simple peer reputation + + GetBlockBodies getBlockBodiesFn + GetNodeData getNodeDataFn + GetReceipts getReceiptsFn + GetProofs getProofsFn + + version int // LES protocol version number +} + +// newPeer creates a new ODR peer +func newPeer(id string, version int, head common.Hash, getBlockBodies getBlockBodiesFn, getNodeData getNodeDataFn, getReceipts getReceiptsFn, getProofs getProofsFn) *Peer { + return &Peer{ + id: id, + head: head, + GetBlockBodies: getBlockBodies, + GetNodeData: getNodeData, + GetReceipts: getReceipts, + GetProofs: getProofs, + version: version, + } +} + +// Id returns a peer's ID string +func (p *Peer) Id() string { + return p.id +} + + +// Promote increases the peer's reputation +func (p *Peer) Promote() { + atomic.AddInt32(&p.rep, 1) +} + +// Demote decreases the peer's reputation or leaves it at 0. +func (p *Peer) Demote() { + for { + // Calculate the new reputation value + prev := atomic.LoadInt32(&p.rep) + next := prev / 2 + + // Try to update the old value + if atomic.CompareAndSwapInt32(&p.rep, prev, next) { + return + } + } +} + +// peerSet represents the collection of active peer participating in the block +// download procedure. +type peerSet struct { + peers map[string]*Peer + lock sync.RWMutex +} + +// newPeerSet creates a new peer set top track the active download sources. +func newPeerSet() *peerSet { + return &peerSet{ + peers: make(map[string]*Peer), + } +} + +// Register injects a new peer into the working set, or returns an error if the +// peer is already known. +func (ps *peerSet) Register(p *Peer) error { + ps.lock.Lock() + defer ps.lock.Unlock() + + if _, ok := ps.peers[p.id]; ok { + return errAlreadyRegistered + } + ps.peers[p.id] = p + return nil +} + +// Unregister removes a remote peer from the active set, disabling any further +// actions to/from that particular entity. +func (ps *peerSet) Unregister(id string) error { + ps.lock.Lock() + defer ps.lock.Unlock() + + if _, ok := ps.peers[id]; !ok { + return errNotRegistered + } + delete(ps.peers, id) + return nil +} + +// Peer retrieves the registered peer with the given id. +func (ps *peerSet) Peer(id string) *Peer { + ps.lock.RLock() + defer ps.lock.RUnlock() + + return ps.peers[id] +} + +// Len returns if the current number of peers in the set. +func (ps *peerSet) Len() int { + ps.lock.RLock() + defer ps.lock.RUnlock() + + return len(ps.peers) +} + +// AllPeers retrieves a flat list of all the peers within the set. +func (ps *peerSet) AllPeers() []*Peer { + ps.lock.RLock() + defer ps.lock.RUnlock() + + list := make([]*Peer, 0, len(ps.peers)) + for _, p := range ps.peers { + list = append(list, p) + } + return list +} + +// BestPeers returns an ordered list of available peers, starting with the +// highest reputation +func (ps *peerSet) BestPeers() []*Peer { + list := ps.AllPeers() + ps.lock.RLock() + defer ps.lock.RUnlock() + + for i := 0; i < len(list); i++ { + for j := i + 1; j < len(list); j++ { + if atomic.LoadInt32(&list[i].rep) < atomic.LoadInt32(&list[j].rep) { + list[i], list[j] = list[j], list[i] + } + } + } + return list +} diff --git a/core/bench_test.go b/core/bench_test.go index b5eb518033..704e6a0ad4 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -168,8 +169,9 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { // Time the insertion of the new chain. // State and blocks are stored in the same DB. evmux := new(event.TypeMux) - chainman, _ := NewBlockChain(db, FakePow{}, evmux) - chainman.SetProcessor(NewBlockProcessor(db, FakePow{}, chainman, evmux)) + ca := access.NewDbChainAccess(db) + chainman, _ := NewBlockChain(ca, FakePow{}, evmux) + chainman.SetProcessor(NewBlockProcessor(ca, FakePow{}, chainman, evmux)) defer chainman.Stop() b.ReportAllocs() b.ResetTimer() diff --git a/core/block_processor.go b/core/block_processor.go index e7b2f63e5a..1511eaf89b 100644 --- a/core/block_processor.go +++ b/core/block_processor.go @@ -23,11 +23,11 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -43,7 +43,7 @@ const ( ) type BlockProcessor struct { - chainDb ethdb.Database + chainAccess *access.ChainAccess // Mutex for locking the block processor. Blocks can only be handled one at a time mutex sync.Mutex // Canonical block chain @@ -85,9 +85,9 @@ func (gp *GasPool) String() string { return (*big.Int)(gp).String() } -func NewBlockProcessor(db ethdb.Database, pow pow.PoW, blockchain *BlockChain, eventMux *event.TypeMux) *BlockProcessor { +func NewBlockProcessor(ca *access.ChainAccess, pow pow.PoW, blockchain *BlockChain, eventMux *event.TypeMux) *BlockProcessor { sm := &BlockProcessor{ - chainDb: db, + chainAccess: ca, mem: make(map[string]*big.Int), Pow: pow, bc: blockchain, @@ -212,12 +212,12 @@ func (sm *BlockProcessor) Process(block *types.Block) (logs vm.Logs, receipts ty defer sm.mutex.Unlock() if sm.bc.HasBlock(block.Hash()) { - if _, err := state.New(block.Root(), sm.chainDb); err == nil { + if _, err := state.New(block.Root(), sm.chainAccess); err == nil { return nil, nil, &KnownBlockError{block.Number(), block.Hash()} } } if parent := sm.bc.GetBlock(block.ParentHash()); parent != nil { - if _, err := state.New(parent.Root(), sm.chainDb); err == nil { + if _, err := state.New(parent.Root(), sm.chainAccess); err == nil { return sm.processWithParent(block, parent) } } @@ -226,7 +226,7 @@ func (sm *BlockProcessor) Process(block *types.Block) (logs vm.Logs, receipts ty func (sm *BlockProcessor) processWithParent(block, parent *types.Block) (logs vm.Logs, receipts types.Receipts, err error) { // Create a new state based on the parent's root (e.g., create copy) - state, err := state.New(parent.Root(), sm.chainDb) + state, err := state.New(parent.Root(), sm.chainAccess) if err != nil { return nil, nil, err } @@ -370,7 +370,7 @@ func (sm *BlockProcessor) VerifyUncles(statedb *state.StateDB, block, parent *ty // GetBlockReceipts returns the receipts beloniging to the block hash func (sm *BlockProcessor) GetBlockReceipts(bhash common.Hash) types.Receipts { if block := sm.BlockChain().GetBlock(bhash); block != nil { - return GetBlockReceipts(sm.chainDb, block.Hash()) + return GetBlockReceipts(sm.chainAccess, block.Hash()) } return nil @@ -380,7 +380,7 @@ func (sm *BlockProcessor) GetBlockReceipts(bhash common.Hash) types.Receipts { // where it tries to get it from the (updated) method which gets them from the receipts or // the depricated way by re-processing the block. func (sm *BlockProcessor) GetLogs(block *types.Block) (logs vm.Logs, err error) { - receipts := GetBlockReceipts(sm.chainDb, block.Hash()) + receipts := GetBlockReceipts(sm.chainAccess, block.Hash()) // coalesce logs for _, receipt := range receipts { logs = append(logs, receipt.Logs...) diff --git a/core/block_processor_test.go b/core/block_processor_test.go index 3050456b41..a5f6527785 100644 --- a/core/block_processor_test.go +++ b/core/block_processor_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -32,21 +33,22 @@ import ( func proc() (*BlockProcessor, *BlockChain) { db, _ := ethdb.NewMemDatabase() + ca := access.NewDbChainAccess(db) var mux event.TypeMux WriteTestNetGenesisBlock(db, 0) - blockchain, err := NewBlockChain(db, thePow(), &mux) + blockchain, err := NewBlockChain(ca, thePow(), &mux) if err != nil { fmt.Println(err) } - return NewBlockProcessor(db, ezp.New(), blockchain, &mux), blockchain + return NewBlockProcessor(ca, ezp.New(), blockchain, &mux), blockchain } func TestNumber(t *testing.T) { pow := ezp.New() _, chain := proc() - statedb, _ := state.New(chain.Genesis().Root(), chain.chainDb) + statedb, _ := state.New(chain.Genesis().Root(), access.NewDbChainAccess(chain.chainDb)) header := makeHeader(chain.Genesis(), statedb) header.Number = big.NewInt(3) err := ValidateHeader(pow, header, chain.Genesis().Header(), false, false) @@ -82,7 +84,7 @@ func TestPutReceipt(t *testing.T) { }} PutReceipts(db, types.Receipts{receipt}) - receipt = GetReceipt(db, common.Hash{}) + receipt = GetReceipt(access.NewDbChainAccess(db), common.Hash{}) if receipt == nil { t.Error("expected to get 1 receipt, got none.") } diff --git a/core/blockchain.go b/core/blockchain.go index cea346e387..4484a6d73b 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -31,6 +31,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -43,6 +44,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/hashicorp/golang-lru" + "golang.org/x/net/context" ) var ( @@ -64,6 +66,7 @@ const ( ) type BlockChain struct { + chainAccess *access.ChainAccess chainDb ethdb.Database processor types.BlockProcessor eventMux *event.TypeMux @@ -95,7 +98,7 @@ type BlockChain struct { rand *mrand.Rand } -func NewBlockChain(chainDb ethdb.Database, pow pow.PoW, mux *event.TypeMux) (*BlockChain, error) { +func NewBlockChain(chainAccess *access.ChainAccess, pow pow.PoW, mux *event.TypeMux) (*BlockChain, error) { headerCache, _ := lru.New(headerCacheLimit) bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) @@ -104,7 +107,8 @@ func NewBlockChain(chainDb ethdb.Database, pow pow.PoW, mux *event.TypeMux) (*Bl futureBlocks, _ := lru.New(maxFutureBlocks) bc := &BlockChain{ - chainDb: chainDb, + chainDb: chainAccess.Db(), + chainAccess: chainAccess, eventMux: mux, quit: make(chan struct{}), headerCache: headerCache, @@ -128,7 +132,7 @@ func NewBlockChain(chainDb ethdb.Database, pow pow.PoW, mux *event.TypeMux) (*Bl if err != nil { return nil, err } - bc.genesisBlock, err = WriteGenesisBlock(chainDb, reader) + bc.genesisBlock, err = WriteGenesisBlock(bc.chainDb, reader) if err != nil { return nil, err } @@ -344,8 +348,14 @@ func (self *BlockChain) SetProcessor(proc types.BlockProcessor) { self.processor = proc } +// State returns a new StateDB for the current block func (self *BlockChain) State() (*state.StateDB, error) { - return state.New(self.CurrentBlock().Root(), self.chainDb) + return state.New(self.CurrentBlock().Root(), self.chainAccess) +} + +// StateOdr returns a new StateDB for the current block with ODR enabled +func (self *BlockChain) StateOdr(ctx context.Context) (*state.StateDB, error) { + return state.NewOdr(ctx, self.CurrentBlock().Root(), self.chainAccess) } // Reset purges the entire blockchain, restoring it to its genesis state. @@ -482,12 +492,18 @@ func (self *BlockChain) GetHeaderByNumber(number uint64) *types.Header { // GetBody retrieves a block body (transactions and uncles) from the database by // hash, caching it if found. func (self *BlockChain) GetBody(hash common.Hash) *types.Body { + return self.GetBodyOdr(access.NoOdr, hash) +} + +// GetBodyOdr retrieves a block body (transactions and uncles) from the database +// or network by hash, caching it if found. +func (self *BlockChain) GetBodyOdr(ctx context.Context, hash common.Hash) *types.Body { // Short circuit if the body's already in the cache, retrieve otherwise if cached, ok := self.bodyCache.Get(hash); ok { body := cached.(*types.Body) return body } - body := GetBody(self.chainDb, hash) + body := GetBodyOdr(ctx, self.chainAccess, hash) if body == nil { return nil } @@ -499,11 +515,17 @@ func (self *BlockChain) GetBody(hash common.Hash) *types.Body { // GetBodyRLP retrieves a block body in RLP encoding from the database by hash, // caching it if found. func (self *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { + return self.GetBodyRLPOdr(access.NoOdr, hash) +} + +// GetBodyRLPOdr retrieves a block body in RLP encoding from the database or +// network by hash, caching it if found. +func (self *BlockChain) GetBodyRLPOdr(ctx context.Context, hash common.Hash) rlp.RawValue { // Short circuit if the body's already in the cache, retrieve otherwise if cached, ok := self.bodyRLPCache.Get(hash); ok { return cached.(rlp.RawValue) } - body := GetBodyRLP(self.chainDb, hash) + body := GetBodyRLPOdr(ctx, self.chainAccess, hash) if len(body) == 0 { return nil } @@ -536,11 +558,16 @@ func (bc *BlockChain) HasBlock(hash common.Hash) bool { // GetBlock retrieves a block from the database by hash, caching it if found. func (self *BlockChain) GetBlock(hash common.Hash) *types.Block { + return self.GetBlockOdr(access.NoOdr, hash) +} + +// GetBlockOdr retrieves a block from the database or network by hash, caching it if found. +func (self *BlockChain) GetBlockOdr(ctx context.Context, hash common.Hash) *types.Block { // Short circuit if the block's already in the cache, retrieve otherwise if block, ok := self.blockCache.Get(hash); ok { return block.(*types.Block) } - block := GetBlock(self.chainDb, hash) + block := GetBlockOdr(ctx, self.chainAccess, hash) if block == nil { return nil } @@ -552,11 +579,17 @@ func (self *BlockChain) GetBlock(hash common.Hash) *types.Block { // GetBlockByNumber retrieves a block from the database by number, caching it // (associated with its hash) if found. func (self *BlockChain) GetBlockByNumber(number uint64) *types.Block { + return self.GetBlockByNumberOdr(access.NoOdr, number) +} + +// GetBlockByNumberOdr retrieves a block from the database or network by number, +// caching it (associated with its hash) if found. +func (self *BlockChain) GetBlockByNumberOdr(ctx context.Context, number uint64) *types.Block { hash := GetCanonicalHash(self.chainDb, number) if hash == (common.Hash{}) { return nil } - return self.GetBlock(hash) + return self.GetBlockOdr(ctx, hash) } // GetBlockHashesFromHash retrieves a number of block hashes starting at a given @@ -1209,7 +1242,7 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { if err := PutTransactions(self.chainDb, block, block.Transactions()); err != nil { return err } - receipts := GetBlockReceipts(self.chainDb, block.Hash()) + receipts := GetBlockReceipts(self.chainAccess, block.Hash()) // write receipts if err := PutReceipts(self.chainDb, receipts); err != nil { return err @@ -1270,4 +1303,4 @@ func blockErr(block *types.Block, err error) { glog.Errorf("Bad block #%v (%s)\n", block.Number(), block.Hash().Hex()) glog.Errorf(" %v", err) } -} +} \ No newline at end of file diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 8ddc5032b1..523b7bcd28 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/ethash" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -51,13 +52,14 @@ func thePow() pow.PoW { func theBlockChain(db ethdb.Database, t *testing.T) *BlockChain { var eventMux event.TypeMux WriteTestNetGenesisBlock(db, 0) - blockchain, err := NewBlockChain(db, thePow(), &eventMux) + ca := access.NewDbChainAccess(db) + blockchain, err := NewBlockChain(ca, thePow(), &eventMux) if err != nil { t.Error("failed creating chainmanager:", err) t.FailNow() return nil } - blockMan := NewBlockProcessor(db, nil, blockchain, &eventMux) + blockMan := NewBlockProcessor(ca, nil, blockchain, &eventMux) blockchain.SetProcessor(blockMan) return blockchain @@ -138,8 +140,8 @@ func testBlockChainImport(chain []*types.Block, processor *BlockProcessor) error } // Manually insert the block into the database, but don't reorganize (allows subsequent testing) processor.bc.mu.Lock() - WriteTd(processor.chainDb, block.Hash(), new(big.Int).Add(block.Difficulty(), processor.bc.GetTd(block.ParentHash()))) - WriteBlock(processor.chainDb, block) + WriteTd(processor.chainAccess.Db(), block.Hash(), new(big.Int).Add(block.Difficulty(), processor.bc.GetTd(block.ParentHash()))) + WriteBlock(processor.chainAccess.Db(), block) processor.bc.mu.Unlock() } return nil @@ -155,8 +157,8 @@ func testHeaderChainImport(chain []*types.Header, processor *BlockProcessor) err } // Manually insert the header into the database, but don't reorganize (allows subsequent testing) processor.bc.mu.Lock() - WriteTd(processor.chainDb, header.Hash(), new(big.Int).Add(header.Difficulty, processor.bc.GetTd(header.ParentHash))) - WriteHeader(processor.chainDb, header) + WriteTd(processor.chainAccess.Db(), header.Hash(), new(big.Int).Add(header.Difficulty, processor.bc.GetTd(header.ParentHash))) + WriteHeader(processor.chainAccess.Db(), header) processor.bc.mu.Unlock() } return nil @@ -452,7 +454,8 @@ func makeBlockChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.B func chm(genesis *types.Block, db ethdb.Database) *BlockChain { var eventMux event.TypeMux - bc := &BlockChain{chainDb: db, genesisBlock: genesis, eventMux: &eventMux, pow: FakePow{}, rand: rand.New(rand.NewSource(0))} + ca := access.NewDbChainAccess(db) + bc := &BlockChain{chainAccess: ca, chainDb: db, genesisBlock: genesis, eventMux: &eventMux, pow: FakePow{}, rand: rand.New(rand.NewSource(0))} bc.headerCache, _ = lru.New(100) bc.bodyCache, _ = lru.New(100) bc.bodyRLPCache, _ = lru.New(100) @@ -500,7 +503,7 @@ func testReorg(t *testing.T, first, second []int, td int64, full bool) { // Check that the chain is valid number and link wise if full { prev := bc.CurrentBlock() - for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) { + for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64()-1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) { if prev.ParentHash() != block.Hash() { t.Errorf("parent block hash mismatch: have %x, want %x", prev.ParentHash(), block.Hash()) } @@ -587,7 +590,7 @@ func testReorgBadHashes(t *testing.T, full bool) { defer func() { delete(BadHashes, headers[3].Hash()) }() } // Create a new chain manager and check it rolled back the state - ncm, err := NewBlockChain(db, FakePow{}, new(event.TypeMux)) + ncm, err := NewBlockChain(access.NewDbChainAccess(db), FakePow{}, new(event.TypeMux)) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } @@ -665,7 +668,7 @@ func testInsertNonceError(t *testing.T, full bool) { // Check that all no blocks after the failing block have been inserted. for j := 0; j < i-failAt; j++ { if full { - if block := bc.GetBlockByNumber(failNum + uint64(j)); block != nil { + if block := bc.GetBlockByNumber(failNum+uint64(j)); block != nil { t.Errorf("test %d: invalid block in chain: %v", i, block) } } else { @@ -708,19 +711,21 @@ func TestFastVsFullChains(t *testing.T) { }) // Import the chain as an archive node for the comparison baseline archiveDb, _ := ethdb.NewMemDatabase() + archiveCa := access.NewDbChainAccess(archiveDb) WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds}) - archive, _ := NewBlockChain(archiveDb, FakePow{}, new(event.TypeMux)) - archive.SetProcessor(NewBlockProcessor(archiveDb, FakePow{}, archive, new(event.TypeMux))) + archive, _ := NewBlockChain(archiveCa, FakePow{}, new(event.TypeMux)) + archive.SetProcessor(NewBlockProcessor(archiveCa, FakePow{}, archive, new(event.TypeMux))) if n, err := archive.InsertChain(blocks); err != nil { t.Fatalf("failed to process block %d: %v", n, err) } // Fast import the chain as a non-archive node to test fastDb, _ := ethdb.NewMemDatabase() + fastCa := access.NewDbChainAccess(fastDb) WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds}) - fast, _ := NewBlockChain(fastDb, FakePow{}, new(event.TypeMux)) - fast.SetProcessor(NewBlockProcessor(fastDb, FakePow{}, fast, new(event.TypeMux))) + fast, _ := NewBlockChain(fastCa, FakePow{}, new(event.TypeMux)) + fast.SetProcessor(NewBlockProcessor(fastCa, FakePow{}, fast, new(event.TypeMux))) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -749,7 +754,7 @@ func TestFastVsFullChains(t *testing.T) { } else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(ablock.Uncles()) { t.Errorf("block #%d [%x]: uncles mismatch: have %v, want %v", num, hash, fblock.Uncles(), ablock.Uncles()) } - if freceipts, areceipts := GetBlockReceipts(fastDb, hash), GetBlockReceipts(archiveDb, hash); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { + if freceipts, areceipts := GetBlockReceipts(fastCa, hash), GetBlockReceipts(archiveCa, hash); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) { t.Errorf("block #%d [%x]: receipts mismatch: have %v, want %v", num, hash, freceipts, areceipts) } } @@ -794,10 +799,10 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { } // Import the chain as an archive node and ensure all pointers are updated archiveDb, _ := ethdb.NewMemDatabase() + archiveCa := access.NewDbChainAccess(archiveDb) WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds}) - - archive, _ := NewBlockChain(archiveDb, FakePow{}, new(event.TypeMux)) - archive.SetProcessor(NewBlockProcessor(archiveDb, FakePow{}, archive, new(event.TypeMux))) + archive, _ := NewBlockChain(archiveCa, FakePow{}, new(event.TypeMux)) + archive.SetProcessor(NewBlockProcessor(archiveCa, FakePow{}, archive, new(event.TypeMux))) if n, err := archive.InsertChain(blocks); err != nil { t.Fatalf("failed to process block %d: %v", n, err) @@ -808,9 +813,10 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a non-archive node and ensure all pointers are updated fastDb, _ := ethdb.NewMemDatabase() + fastCa := access.NewDbChainAccess(fastDb) WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds}) - fast, _ := NewBlockChain(fastDb, FakePow{}, new(event.TypeMux)) - fast.SetProcessor(NewBlockProcessor(fastDb, FakePow{}, fast, new(event.TypeMux))) + fast, _ := NewBlockChain(fastCa, FakePow{}, new(event.TypeMux)) + fast.SetProcessor(NewBlockProcessor(fastCa, FakePow{}, fast, new(event.TypeMux))) headers := make([]*types.Header, len(blocks)) for i, block := range blocks { @@ -828,9 +834,10 @@ func TestLightVsFastVsFullChainHeads(t *testing.T) { // Import the chain as a light node and ensure all pointers are updated lightDb, _ := ethdb.NewMemDatabase() + lightCa := access.NewDbChainAccess(lightDb) WriteGenesisBlockForTesting(lightDb, GenesisAccount{address, funds}) - light, _ := NewBlockChain(lightDb, FakePow{}, new(event.TypeMux)) - light.SetProcessor(NewBlockProcessor(lightDb, FakePow{}, light, new(event.TypeMux))) + light, _ := NewBlockChain(lightCa, FakePow{}, new(event.TypeMux)) + light.SetProcessor(NewBlockProcessor(lightCa, FakePow{}, light, new(event.TypeMux))) if n, err := light.InsertHeaderChain(headers, 1); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) @@ -895,8 +902,9 @@ func TestChainTxReorgs(t *testing.T) { }) // Import the chain. This runs all block validation rules. evmux := &event.TypeMux{} - chainman, _ := NewBlockChain(db, FakePow{}, evmux) - chainman.SetProcessor(NewBlockProcessor(db, FakePow{}, chainman, evmux)) + ca := access.NewDbChainAccess(db) + chainman, _ := NewBlockChain(ca, FakePow{}, evmux) + chainman.SetProcessor(NewBlockProcessor(ca, FakePow{}, chainman, evmux)) if i, err := chainman.InsertChain(chain); err != nil { t.Fatalf("failed to insert original chain[%d]: %v", i, err) } @@ -929,7 +937,7 @@ func TestChainTxReorgs(t *testing.T) { if GetTransaction(db, tx.Hash()) != nil { t.Errorf("drop %d: tx found while shouldn't have been", i) } - if GetReceipt(db, tx.Hash()) != nil { + if GetReceipt(ca, tx.Hash()) != nil { t.Errorf("drop %d: receipt found while shouldn't have been", i) } } @@ -938,7 +946,7 @@ func TestChainTxReorgs(t *testing.T) { if GetTransaction(db, tx.Hash()) == nil { t.Errorf("add %d: expected tx to be found", i) } - if GetReceipt(db, tx.Hash()) == nil { + if GetReceipt(ca, tx.Hash()) == nil { t.Errorf("add %d: expected receipt to be found", i) } } @@ -947,7 +955,7 @@ func TestChainTxReorgs(t *testing.T) { if GetTransaction(db, tx.Hash()) == nil { t.Errorf("share %d: expected tx to be found", i) } - if GetReceipt(db, tx.Hash()) == nil { + if GetReceipt(ca, tx.Hash()) == nil { t.Errorf("share %d: expected receipt to be found", i) } } diff --git a/core/chain_makers.go b/core/chain_makers.go index 56e37a0fc2..e6f69541f9 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -21,6 +21,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -164,7 +165,7 @@ func (b *BlockGen) OffsetTime(seconds int64) { // values. Inserting them into BlockChain requires use of FakePow or // a similar non-validating proof of work implementation. func GenerateChain(parent *types.Block, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts) { - statedb, err := state.New(parent.Root(), db) + statedb, err := state.New(parent.Root(), access.NewDbChainAccess(db)) if err != nil { panic(err) } @@ -222,8 +223,9 @@ func newCanonical(n int, full bool) (ethdb.Database, *BlockProcessor, error) { // Initialize a fresh chain with only a genesis block genesis, _ := WriteTestNetGenesisBlock(db, 0) - blockchain, _ := NewBlockChain(db, FakePow{}, evmux) - processor := NewBlockProcessor(db, FakePow{}, blockchain, evmux) + ca := access.NewDbChainAccess(db) + blockchain, _ := NewBlockChain(ca, FakePow{}, evmux) + processor := NewBlockProcessor(ca, FakePow{}, blockchain, evmux) processor.bc.SetProcessor(processor) // Create and inject the requested chain diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index 7f47cf2888..6e87dd0ff0 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -20,6 +20,7 @@ import ( "fmt" "math/big" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -77,8 +78,9 @@ func ExampleGenerateChain() { // Import the chain. This runs all block validation rules. evmux := &event.TypeMux{} - chainman, _ := NewBlockChain(db, FakePow{}, evmux) - chainman.SetProcessor(NewBlockProcessor(db, FakePow{}, chainman, evmux)) + ca := access.NewDbChainAccess(db) + chainman, _ := NewBlockChain(ca, FakePow{}, evmux) + chainman.SetProcessor(NewBlockProcessor(ca, FakePow{}, chainman, evmux)) if i, err := chainman.InsertChain(chain); err != nil { fmt.Printf("insert error (block %d): %v\n", i, err) return diff --git a/core/chain_util.go b/core/chain_util.go index ddff381a1d..439d882e5f 100644 --- a/core/chain_util.go +++ b/core/chain_util.go @@ -23,12 +23,15 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" + "github.com/ethereum/go-ethereum/core/requests" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/net/context" ) var ( @@ -182,16 +185,31 @@ func GetHeader(db ethdb.Database, hash common.Hash) *types.Header { return header } -// GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func GetBodyRLP(db ethdb.Database, hash common.Hash) rlp.RawValue { - data, _ := db.Get(append(append(blockPrefix, hash[:]...), bodySuffix...)) - return data +// GetBodyRLP retrieves the block body (transactions and uncles) from the +// database in RLP encoding. +func GetBodyRLP(ca *access.ChainAccess, hash common.Hash) rlp.RawValue { + return GetBodyRLPOdr(access.NoOdr, ca, hash) +} + +// GetBodyRLPOdr retrieves the block body (transactions and uncles) from the +// database or network in RLP encoding. +func GetBodyRLPOdr(ctx context.Context, ca *access.ChainAccess, hash common.Hash) rlp.RawValue { + //fmt.Println("request block %v", hash) + r := requests.NewBlockAccess(ca.Db(), hash, GetHeader) + ca.Retrieve(ctx, r) + return r.GetRlp() } // GetBody retrieves the block body (transactons, uncles) corresponding to the -// hash, nil if none found. -func GetBody(db ethdb.Database, hash common.Hash) *types.Body { - data := GetBodyRLP(db, hash) +// hash from the database, nil if none found. +func GetBody(ca *access.ChainAccess, hash common.Hash) *types.Body { + return GetBodyOdr(access.NoOdr, ca, hash) +} + +// GetBodyOdr retrieves the block body (transactons, uncles) corresponding to the +// hash from the database or network, nil if none found. +func GetBodyOdr(ctx context.Context, ca *access.ChainAccess, hash common.Hash) *types.Body { + data := GetBodyRLPOdr(ctx, ca, hash) if len(data) == 0 { return nil } @@ -218,15 +236,21 @@ func GetTd(db ethdb.Database, hash common.Hash) *big.Int { return td } -// GetBlock retrieves an entire block corresponding to the hash, assembling it -// back from the stored header and body. -func GetBlock(db ethdb.Database, hash common.Hash) *types.Block { +// GetBlock retrieves an entire block from the database corresponding to the +// hash, assembling it back from the stored header and body. +func GetBlock(ca *access.ChainAccess, hash common.Hash) *types.Block { + return GetBlockOdr(access.NoOdr, ca, hash) +} + +// GetBlockOdr retrieves an entire block from database or network corresponding +// to the hash, assembling it back from the stored header and body. +func GetBlockOdr(ctx context.Context, ca *access.ChainAccess, hash common.Hash) *types.Block { // Retrieve the block header and body contents - header := GetHeader(db, hash) + header := GetHeader(ca.Db(), hash) if header == nil { return nil } - body := GetBody(db, hash) + body := GetBodyOdr(ctx, ca, hash) if body == nil { return nil } diff --git a/core/chain_util_test.go b/core/chain_util_test.go index 0bbcbbe53d..53bcfaf8f2 100644 --- a/core/chain_util_test.go +++ b/core/chain_util_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -121,6 +122,7 @@ func TestHeaderStorage(t *testing.T) { // Tests block body storage and retrieval operations. func TestBodyStorage(t *testing.T) { db, _ := ethdb.NewMemDatabase() + ca := access.NewDbChainAccess(db) // Create a test body to move around the database and make sure it's really new body := &types.Body{Uncles: []*types.Header{{Extra: []byte("test header")}}} @@ -129,19 +131,19 @@ func TestBodyStorage(t *testing.T) { rlp.Encode(hasher, body) hash := common.BytesToHash(hasher.Sum(nil)) - if entry := GetBody(db, hash); entry != nil { + if entry := GetBody(ca, hash); entry != nil { t.Fatalf("Non existent body returned: %v", entry) } // Write and verify the body in the database if err := WriteBody(db, hash, body); err != nil { t.Fatalf("Failed to write body into database: %v", err) } - if entry := GetBody(db, hash); entry == nil { + if entry := GetBody(ca, hash); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(types.Transactions(body.Transactions)) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(body.Uncles) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, body) } - if entry := GetBodyRLP(db, hash); entry == nil { + if entry := GetBodyRLP(ca, hash); entry == nil { t.Fatalf("Stored body RLP not found") } else { hasher := sha3.NewKeccak256() @@ -153,7 +155,7 @@ func TestBodyStorage(t *testing.T) { } // Delete the body and verify the execution DeleteBody(db, hash) - if entry := GetBody(db, hash); entry != nil { + if entry := GetBody(ca, hash); entry != nil { t.Fatalf("Deleted body returned: %v", entry) } } @@ -161,6 +163,7 @@ func TestBodyStorage(t *testing.T) { // Tests block storage and retrieval operations. func TestBlockStorage(t *testing.T) { db, _ := ethdb.NewMemDatabase() + ca := access.NewDbChainAccess(db) // Create a test block to move around the database and make sure it's really new block := types.NewBlockWithHeader(&types.Header{ @@ -169,20 +172,20 @@ func TestBlockStorage(t *testing.T) { TxHash: types.EmptyRootHash, ReceiptHash: types.EmptyRootHash, }) - if entry := GetBlock(db, block.Hash()); entry != nil { + if entry := GetBlock(ca, block.Hash()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } if entry := GetHeader(db, block.Hash()); entry != nil { t.Fatalf("Non existent header returned: %v", entry) } - if entry := GetBody(db, block.Hash()); entry != nil { + if entry := GetBody(ca, block.Hash()); entry != nil { t.Fatalf("Non existent body returned: %v", entry) } // Write and verify the block in the database if err := WriteBlock(db, block); err != nil { t.Fatalf("Failed to write block into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry == nil { + if entry := GetBlock(ca, block.Hash()); entry == nil { t.Fatalf("Stored block not found") } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) @@ -192,20 +195,20 @@ func TestBlockStorage(t *testing.T) { } else if entry.Hash() != block.Header().Hash() { t.Fatalf("Retrieved header mismatch: have %v, want %v", entry, block.Header()) } - if entry := GetBody(db, block.Hash()); entry == nil { + if entry := GetBody(ca, block.Hash()); entry == nil { t.Fatalf("Stored body not found") } else if types.DeriveSha(types.Transactions(entry.Transactions)) != types.DeriveSha(block.Transactions()) || types.CalcUncleHash(entry.Uncles) != types.CalcUncleHash(block.Uncles()) { t.Fatalf("Retrieved body mismatch: have %v, want %v", entry, &types.Body{block.Transactions(), block.Uncles()}) } // Delete the block and verify the execution DeleteBlock(db, block.Hash()) - if entry := GetBlock(db, block.Hash()); entry != nil { + if entry := GetBlock(ca, block.Hash()); entry != nil { t.Fatalf("Deleted block returned: %v", entry) } if entry := GetHeader(db, block.Hash()); entry != nil { t.Fatalf("Deleted header returned: %v", entry) } - if entry := GetBody(db, block.Hash()); entry != nil { + if entry := GetBody(ca, block.Hash()); entry != nil { t.Fatalf("Deleted body returned: %v", entry) } } @@ -213,6 +216,7 @@ func TestBlockStorage(t *testing.T) { // Tests that partial block contents don't get reassembled into full blocks. func TestPartialBlockStorage(t *testing.T) { db, _ := ethdb.NewMemDatabase() + ca := access.NewDbChainAccess(db) block := types.NewBlockWithHeader(&types.Header{ Extra: []byte("test block"), UncleHash: types.EmptyUncleHash, @@ -223,7 +227,7 @@ func TestPartialBlockStorage(t *testing.T) { if err := WriteHeader(db, block.Header()); err != nil { t.Fatalf("Failed to write header into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry != nil { + if entry := GetBlock(ca, block.Hash()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } DeleteHeader(db, block.Hash()) @@ -232,7 +236,7 @@ func TestPartialBlockStorage(t *testing.T) { if err := WriteBody(db, block.Hash(), &types.Body{block.Transactions(), block.Uncles()}); err != nil { t.Fatalf("Failed to write body into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry != nil { + if entry := GetBlock(ca, block.Hash()); entry != nil { t.Fatalf("Non existent block returned: %v", entry) } DeleteBody(db, block.Hash()) @@ -244,7 +248,7 @@ func TestPartialBlockStorage(t *testing.T) { if err := WriteBody(db, block.Hash(), &types.Body{block.Transactions(), block.Uncles()}); err != nil { t.Fatalf("Failed to write body into database: %v", err) } - if entry := GetBlock(db, block.Hash()); entry == nil { + if entry := GetBlock(ca, block.Hash()); entry == nil { t.Fatalf("Stored block not found") } else if entry.Hash() != block.Hash() { t.Fatalf("Retrieved block mismatch: have %v, want %v", entry, block) diff --git a/core/genesis.go b/core/genesis.go index dac5de92ff..c5a355f0c0 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" @@ -61,7 +62,8 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, } // creating with empty hash always works - statedb, _ := state.New(common.Hash{}, chainDb) + ca := access.NewDbChainAccess(chainDb) + statedb, _ := state.New(common.Hash{}, ca) for addr, account := range genesis.Alloc { address := common.HexToAddress(addr) statedb.AddBalance(address, common.String2Big(account.Balance)) @@ -85,7 +87,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, Root: root, }, nil, nil, nil) - if block := GetBlock(chainDb, block.Hash()); block != nil { + if block := GetBlock(ca, block.Hash()); block != nil { glog.V(logger.Info).Infoln("Genesis block already in chain. Writing canonical number") err := WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()) if err != nil { @@ -118,7 +120,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block, // GenesisBlockForTesting creates a block in which addr has the given wei balance. // The state trie of the block is written to db. the passed db needs to contain a state root func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big.Int) *types.Block { - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, access.NewDbChainAccess(db)) obj := statedb.GetOrNewStateObject(addr) obj.SetBalance(balance) root, err := statedb.Commit() diff --git a/core/manager.go b/core/manager.go index 289c87c112..c2dbe8bfef 100644 --- a/core/manager.go +++ b/core/manager.go @@ -18,6 +18,7 @@ package core import ( "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" ) @@ -29,6 +30,7 @@ type Backend interface { BlockChain() *BlockChain TxPool() *TxPool ChainDb() ethdb.Database + ChainAccess() *access.ChainAccess DappDb() ethdb.Database EventMux() *event.TypeMux } diff --git a/core/requests/block_access.go b/core/requests/block_access.go new file mode 100644 index 0000000000..50faabe335 --- /dev/null +++ b/core/requests/block_access.go @@ -0,0 +1,204 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package requests + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/core/access" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/rlp" +) + +var ( + blockReceiptsPre = []byte("receipts-block-") + blockPrefix = []byte("block-") + bodySuffix = []byte("-body") +) + +// BlockAccess is the ODR request type for block bodies +type BlockAccess struct { + access.ObjectAccess + db ethdb.Database + blockHash common.Hash + rlp []byte + getHeader getHeaderFn +} + +type getHeaderFn func(db ethdb.Database, hash common.Hash) *types.Header + +// NewBlockAccess creates a new BlockAccess request +func NewBlockAccess(db ethdb.Database, blockHash common.Hash, getHeader getHeaderFn) *BlockAccess { + return &BlockAccess{db: db, blockHash: blockHash, getHeader: getHeader} +} + +// GetRlp returns the block body rlp data stored in memory after a successful request +func (self *BlockAccess) GetRlp() []byte { + return self.rlp +} + +// Request sends an ODR request to the LES network (implementation of access.ObjectAccess) +func (self *BlockAccess) Request(peer *access.Peer) error { + glog.V(access.LogLevel).Infof("ODR: requesting body of block %08x from peer %v", self.blockHash[:4], peer.Id()) + return peer.GetBlockBodies([]common.Hash{self.blockHash}) +} + +// Valid processes an ODR request reply message from the LES network +// returns true and stores results in memory if the message was a valid reply +// to the request (implementation of access.ObjectAccess) +func (self *BlockAccess) Valid(msg *access.Msg) bool { + glog.V(access.LogLevel).Infof("ODR: validating body of block %08x", self.blockHash[:4]) + if msg.MsgType != access.MsgBlockBodies { + glog.V(access.LogLevel).Infof("ODR: invalid message type") + return false + } + bodies := msg.Obj.([]*types.Body) + if len(bodies) != 1 { + glog.V(access.LogLevel).Infof("ODR: invalid number of entries: %d", len(bodies)) + return false + } + body := bodies[0] + header := self.getHeader(self.db, self.blockHash) + if header == nil { + glog.V(access.LogLevel).Infof("ODR: header not found for block %08x", self.blockHash[:4]) + return false + } + txHash := types.DeriveSha(types.Transactions(body.Transactions)) + if header.TxHash != txHash { + glog.V(access.LogLevel).Infof("ODR: header.TxHash %08x does not match received txHash %08x", header.TxHash[:4], txHash[:4]) + return false + } + uncleHash := types.CalcUncleHash(body.Uncles) + if header.UncleHash != uncleHash { + glog.V(access.LogLevel).Infof("ODR: header.UncleHash %08x does not match received uncleHash %08x", header.UncleHash[:4], uncleHash[:4]) + return false + } + data, err := rlp.EncodeToBytes(body) + if err != nil { + glog.V(access.LogLevel).Infof("ODR: body RLP encode error: %v", err) + return false + } + self.rlp = data + glog.V(access.LogLevel).Infof("ODR: validation successful") + return true +} + +// DbGet tries to retrieve requested data from the local database, returns +// true and stores results in memory if successful +// (implementation of access.ObjectAccess) +func (self *BlockAccess) DbGet() bool { + self.rlp, _ = self.db.Get(append(append(blockPrefix, self.blockHash[:]...), bodySuffix...)) + glog.V(access.LogLevel).Infof("ODR: get body %08x len = %d", self.blockHash[:4], len(self.rlp)) + return len(self.rlp) != 0 +} + +// DbPut stores the results of a successful request in the local database +// (implementation of access.ObjectAccess) +func (self *BlockAccess) DbPut() { + self.db.Put(append(append(blockPrefix, self.blockHash[:]...), bodySuffix...), self.rlp) + glog.V(access.LogLevel).Infof("ODR: put body %08x len = %d", self.blockHash[:4], len(self.rlp)) +} + +// ReceiptsAccess is the ODR request type for block receipts by block hash +type ReceiptsAccess struct { + access.ObjectAccess + db ethdb.Database + blockHash common.Hash + receipts types.Receipts + getHeader getHeaderFn + putReceipts putReceiptsFn + putBlockReceipts putBlockReceiptsFn +} + +type putReceiptsFn func(db ethdb.Database, receipts types.Receipts) error +type putBlockReceiptsFn func(db ethdb.Database, hash common.Hash, receipts types.Receipts) error + +// NewReceiptsAccess creates a new ReceiptsAccess request +func NewReceiptsAccess(db ethdb.Database, blockHash common.Hash, getHeader getHeaderFn, putReceipts putReceiptsFn, putBlockReceipts putBlockReceiptsFn) *ReceiptsAccess { + return &ReceiptsAccess{db: db, blockHash: blockHash, getHeader: getHeader, putReceipts: putReceipts, putBlockReceipts: putBlockReceipts} +} + +// GetReceipts returns the block receipts stored in memory after a successful request +func (self *ReceiptsAccess) GetReceipts() types.Receipts { + return self.receipts +} + +// Request sends an ODR request to the LES network (implementation of access.ObjectAccess) +func (self *ReceiptsAccess) Request(peer *access.Peer) error { + glog.V(access.LogLevel).Infof("ODR: requesting receipts for block %08x from peer %v", self.blockHash[:4], peer.Id()) + return peer.GetReceipts([]common.Hash{self.blockHash}) +} + +// Valid processes an ODR request reply message from the LES network +// returns true and stores results in memory if the message was a valid reply +// to the request (implementation of access.ObjectAccess) +func (self *ReceiptsAccess) Valid(msg *access.Msg) bool { + glog.V(access.LogLevel).Infof("ODR: validating receipts for block %08x", self.blockHash[:4]) + if msg.MsgType != access.MsgReceipts { + glog.V(access.LogLevel).Infof("ODR: invalid message type") + return false + } + receipts := msg.Obj.([]types.Receipts) + if len(receipts) != 1 { + glog.V(access.LogLevel).Infof("ODR: invalid number of entries: %d", len(receipts)) + return false + } + hash := types.DeriveSha(receipts[0]) + header := self.getHeader(self.db, self.blockHash) + if header == nil { + glog.V(access.LogLevel).Infof("ODR: header not found for block %08x", self.blockHash[:4]) + return false + } + if !bytes.Equal(header.ReceiptHash[:], hash[:]) { + glog.V(access.LogLevel).Infof("ODR: header receipts hash %08x does not match calculated RLP hash %08x", header.ReceiptHash[:4], hash[:4]) + return false + } + self.receipts = receipts[0] + glog.V(access.LogLevel).Infof("ODR: validation successful") + return true +} + +// DbGet tries to retrieve requested data from the local database, returns +// true and stores results in memory if successful +// (implementation of access.ObjectAccess) +func (self *ReceiptsAccess) DbGet() bool { + data, _ := self.db.Get(append(blockReceiptsPre, self.blockHash[:]...)) + if len(data) == 0 { + return false + } + rs := []*types.ReceiptForStorage{} + if err := rlp.DecodeBytes(data, &rs); err != nil { + glog.V(logger.Error).Infof("invalid receipt array RLP for hash %x: %v", self.blockHash, err) + return false + } + self.receipts = make(types.Receipts, len(rs)) + for i, receipt := range rs { + self.receipts[i] = (*types.Receipt)(receipt) + } + return true +} + +// DbPut stores the results of a successful request in the local database +// (implementation of access.ObjectAccess) +func (self *ReceiptsAccess) DbPut() { + self.putBlockReceipts(self.db, self.blockHash, self.receipts) + self.putReceipts(self.db, self.receipts) +} diff --git a/core/requests/trie_access.go b/core/requests/trie_access.go new file mode 100644 index 0000000000..fd49994bac --- /dev/null +++ b/core/requests/trie_access.go @@ -0,0 +1,200 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package state provides a caching layer atop the Ethereum state trie. +package requests + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/core/access" + "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/trie" + "golang.org/x/net/context" +) + +// TrieAccess implements trie.OdrAccess, providing database/network access for +// a trie identified by root hash +type TrieAccess struct { + trie.OdrAccess + ca *access.ChainAccess + root common.Hash + trieDb trie.Database +} + +// NewTrieAccess creates a new TrieAccess +func NewTrieAccess(ca *access.ChainAccess, root common.Hash, trieDb trie.Database) *TrieAccess { + return &TrieAccess{ + ca: ca, + root: root, + trieDb: trieDb, + } +} + +// RetrieveKey retrieves a single key, returns true and stores nodes in local +// database if successful +func (self *TrieAccess) RetrieveKey(ctx context.Context, key []byte) bool { + //fmt.Println("request trie %v key %v", self.root, key) + r := NewTrieEntryAccess(self.root, self.trieDb, key) + return self.ca.Retrieve(ctx, r) == nil +} + +// OdrEnabled returns true if this TrieAccess is capable of doing network requests +func (self *TrieAccess) OdrEnabled() bool { + return self.ca.OdrEnabled() +} + +// TrieEntryAccess is the ODR request type for state/storage trie entries +type TrieEntryAccess struct { + access.ObjectAccess + root common.Hash + trieDb trie.Database + key, value []byte + proof trie.MerkleProof + skipLevels int // set by DbGet() if unsuccessful +} + +// NewTrieEntryAccess creates a new TrieEntryAccess request +func NewTrieEntryAccess(root common.Hash, trieDb trie.Database, key []byte) *TrieEntryAccess { + return &TrieEntryAccess{root: root, trieDb: trieDb, key: key} +} + +// Request sends an ODR request to the LES network (implementation of access.ObjectAccess) +func (self *TrieEntryAccess) Request(peer *access.Peer) error { + glog.V(access.LogLevel).Infof("ODR: requesting trie root %08x key %08x from peer %v", self.root[:4], self.key[:4], peer.Id()) + req := &access.ProofReq{ + Root: self.root, + Key: self.key, + } + return peer.GetProofs([]*access.ProofReq{req}) +} + +// Valid processes an ODR request reply message from the LES network +// returns true and stores results in memory if the message was a valid reply +// to the request (implementation of access.ObjectAccess) +func (self *TrieEntryAccess) Valid(msg *access.Msg) bool { + glog.V(access.LogLevel).Infof("ODR: validating trie root %08x key %08x", self.root[:4], self.key[:4]) + + if msg.MsgType != access.MsgProofs { + glog.V(access.LogLevel).Infof("ODR: invalid message type") + return false + } + proofs := msg.Obj.([]trie.MerkleProof) + if len(proofs) != 1 { + glog.V(access.LogLevel).Infof("ODR: invalid number of entries: %d", len(proofs)) + return false + } + value, err := trie.VerifyProof(self.root, self.key, proofs[0]) + if err != nil { + glog.V(access.LogLevel).Infof("ODR: merkle proof verification error: %v", err) + return false + } + self.proof = proofs[0] + self.value = value + glog.V(access.LogLevel).Infof("ODR: validation successful") + return true +} + +// DbGet tries to retrieve requested data from the local database, returns +// true and stores results in memory if successful +// (implementation of access.ObjectAccess) +func (self *TrieEntryAccess) DbGet() bool { + return false // not used +} + +// DbPut stores the results of a successful request in the local database +// (implementation of access.ObjectAccess) +func (self *TrieEntryAccess) DbPut() { + trie.StoreProof(self.trieDb, self.proof) +} + +// NodeDataBlockAccess is the ODR request type for node data (used for retrieving contract code) +type NodeDataAccess struct { + access.ObjectAccess + db ethdb.Database + hash common.Hash + data []byte +} + +// NewNodeDataAccess creates a new NodeDataAccess request +func NewNodeDataAccess(db ethdb.Database, hash common.Hash) *NodeDataAccess { + return &NodeDataAccess{db: db, hash: hash} +} + +// Request sends an ODR request to the LES network (implementation of access.ObjectAccess) +func (self *NodeDataAccess) Request(peer *access.Peer) error { + glog.V(access.LogLevel).Infof("ODR: requesting node data for hash %08x from peer %v", self.hash[:4], peer.Id()) + return peer.GetNodeData([]common.Hash{self.hash}) +} + +// Valid processes an ODR request reply message from the LES network +// returns true and stores results in memory if the message was a valid reply +// to the request (implementation of access.ObjectAccess) +func (self *NodeDataAccess) Valid(msg *access.Msg) bool { + glog.V(access.LogLevel).Infof("ODR: validating node data for hash %08x", self.hash[:4]) + if msg.MsgType != access.MsgNodeData { + glog.V(access.LogLevel).Infof("ODR: invalid message type") + return false + } + reply := msg.Obj.([][]byte) + if len(reply) != 1 { + glog.V(access.LogLevel).Infof("ODR: invalid number of entries: %d", len(reply)) + return false + } + data := reply[0] + hash := crypto.Sha3Hash(data) + if bytes.Compare(self.hash[:], hash[:]) != 0 { + glog.V(access.LogLevel).Infof("ODR: requested hash %08x does not match received data hash %08x", self.hash[:4], hash[:4]) + return false + } + self.data = data + glog.V(access.LogLevel).Infof("ODR: validation successful") + return true +} + +// DbGet tries to retrieve requested data from the local database, returns +// true and stores results in memory if successful +// (implementation of access.ObjectAccess) +func (self *NodeDataAccess) DbGet() bool { + data, _ := self.db.Get(self.hash[:]) + if len(data) == 0 { + return false + } + self.data = data + return true +} + +// DbPut stores the results of a successful request in the local database +// (implementation of access.ObjectAccess) +func (self *NodeDataAccess) DbPut() { + self.db.Put(self.hash[:], self.data) +} + +var sha3_nil = sha3.NewKeccak256().Sum(nil) + +func RetrieveNodeData(ctx context.Context, ca *access.ChainAccess, hash common.Hash) []byte { + //fmt.Println("request node data %v", hash) + if bytes.Compare(hash[:], sha3_nil) == 0 { + return nil + } + r := NewNodeDataAccess(ca.Db(), hash) + ca.Retrieve(ctx, r) + return r.data +} diff --git a/core/state/dump.go b/core/state/dump.go index 9acb8a0244..14e7e6ee42 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" ) type Account struct { @@ -45,7 +46,7 @@ func (self *StateDB) RawDump() World { it := self.trie.Iterator() for it.Next() { addr := self.trie.GetKey(it.Key) - stateObject := NewStateObjectFromBytes(common.BytesToAddress(addr), it.Value, self.db) + stateObject := NewStateObjectFromBytes(access.NoOdr, common.BytesToAddress(addr), it.Value, self.ca) account := Account{Balance: stateObject.balance.String(), Nonce: stateObject.nonce, Root: common.Bytes2Hex(stateObject.Root()), CodeHash: common.Bytes2Hex(stateObject.codeHash)} account.Storage = make(map[string]string) diff --git a/core/state/managed_state_test.go b/core/state/managed_state_test.go index 0b53a42c54..5356b0533c 100644 --- a/core/state/managed_state_test.go +++ b/core/state/managed_state_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/ethdb" ) @@ -27,7 +28,7 @@ var addr = common.BytesToAddress([]byte("test")) func create() (*ManagedState, *account) { db, _ := ethdb.NewMemDatabase() - statedb, _ := New(common.Hash{}, db) + statedb, _ := New(common.Hash{}, access.NewDbChainAccess(db)) ms := ManageState(statedb) so := &StateObject{address: addr, nonce: 100} ms.StateDB.stateObjects[addr.Str()] = so diff --git a/core/state/state_object.go b/core/state/state_object.go index c06e3d2276..7fffa071d8 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -22,12 +22,14 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" + "github.com/ethereum/go-ethereum/core/requests" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "golang.org/x/net/context" ) type Code []byte @@ -57,7 +59,8 @@ func (self Storage) Copy() Storage { type StateObject struct { // State database for storing state changes - db ethdb.Database + ca *access.ChainAccess + ctx context.Context trie *trie.SecureTrie // Address belonging to this account @@ -83,14 +86,14 @@ type StateObject struct { dirty bool } -func NewStateObject(address common.Address, db ethdb.Database) *StateObject { - object := &StateObject{db: db, address: address, balance: new(big.Int), dirty: true} - object.trie, _ = trie.NewSecure(common.Hash{}, db) +func NewStateObject(ctx context.Context, address common.Address, ca *access.ChainAccess) *StateObject { + object := &StateObject{ca: ca, ctx: ctx, address: address, balance: new(big.Int), dirty: true} + object.trie, _ = trie.NewSecure(common.Hash{}, ca.Db()) object.storage = make(Storage) return object } -func NewStateObjectFromBytes(address common.Address, data []byte, db ethdb.Database) *StateObject { +func NewStateObjectFromBytes(ctx context.Context, address common.Address, data []byte, ca *access.ChainAccess) *StateObject { var extobject struct { Nonce uint64 Balance *big.Int @@ -102,20 +105,20 @@ func NewStateObjectFromBytes(address common.Address, data []byte, db ethdb.Datab glog.Errorf("can't decode state object %x: %v", address, err) return nil } - trie, err := trie.NewSecure(extobject.Root, db) + trie, err := trie.NewSecureOdr(ctx, extobject.Root, ca.Db(), requests.NewTrieAccess(ca, extobject.Root, ca.Db())) if err != nil { // TODO: bubble this up or panic glog.Errorf("can't create account trie with root %x: %v", extobject.Root[:], err) return nil } - object := &StateObject{address: address, db: db} + object := &StateObject{address: address, ca: ca, ctx: ctx} object.nonce = extobject.Nonce object.balance = extobject.Balance object.codeHash = extobject.CodeHash object.trie = trie object.storage = make(map[string]common.Hash) - object.code, _ = db.Get(extobject.CodeHash) + object.code = requests.RetrieveNodeData(ctx, ca, common.BytesToHash(extobject.CodeHash)) return object } @@ -130,7 +133,8 @@ func (self *StateObject) MarkForDeletion() { func (c *StateObject) getAddr(addr common.Hash) common.Hash { var ret []byte - rlp.DecodeBytes(c.trie.Get(addr[:]), &ret) + value := c.trie.Get(addr[:]) + rlp.DecodeBytes(value, &ret) return common.BytesToHash(ret) } @@ -205,12 +209,22 @@ func (c *StateObject) St() Storage { // Return the gas back to the origin. Used by the Virtual machine or Closures func (c *StateObject) ReturnGas(gas, price *big.Int) {} +// Copy creates a copy of the state object func (self *StateObject) Copy() *StateObject { - stateObject := NewStateObject(self.Address(), self.db) + return self.CopyOdr(access.NoOdr) +} + +// CopyOdr creates a copy of the state object with ODR option +func (self *StateObject) CopyOdr(ctx context.Context) *StateObject { + stateObject := NewStateObject(ctx, self.Address(), self.ca) stateObject.balance.Set(self.balance) stateObject.codeHash = common.CopyBytes(self.codeHash) stateObject.nonce = self.nonce - stateObject.trie = self.trie + if access.IsOdrContext(ctx) { + stateObject.trie = self.trie.CopySecureWithOdr(ctx, requests.NewTrieAccess(self.ca, common.BytesToHash(self.trie.Root()), self.ca.Db())) + } else { + stateObject.trie = self.trie + } stateObject.code = common.CopyBytes(self.code) stateObject.initCode = common.CopyBytes(self.initCode) stateObject.storage = self.storage.Copy() diff --git a/core/state/state_test.go b/core/state/state_test.go index 7ddbe11a13..02e422fa89 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -24,6 +24,7 @@ import ( checker "gopkg.in/check.v1" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/ethdb" ) @@ -77,12 +78,12 @@ func (s *StateSuite) TestDump(c *checker.C) { func (s *StateSuite) SetUpTest(c *checker.C) { db, _ := ethdb.NewMemDatabase() - s.state, _ = New(common.Hash{}, db) + s.state, _ = New(common.Hash{}, access.NewDbChainAccess(db)) } func TestNull(t *testing.T) { db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, db) + state, _ := New(common.Hash{}, access.NewDbChainAccess(db)) address := common.HexToAddress("0x823140710bf13990e4500136726d8b55") state.CreateAccount(address) @@ -122,7 +123,7 @@ func (s *StateSuite) TestSnapshot(c *checker.C) { // printing/logging in tests (-check.vv does not work) func TestSnapshot2(t *testing.T) { db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, db) + state, _ := New(common.Hash{}, access.NewDbChainAccess(db)) stateobjaddr0 := toAddr([]byte("so0")) stateobjaddr1 := toAddr([]byte("so1")) diff --git a/core/state/statedb.go b/core/state/statedb.go index a9de71409c..7e2242fec3 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -22,10 +22,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/core/access" + "github.com/ethereum/go-ethereum/core/requests" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/trie" + "golang.org/x/net/context" ) // The starting nonce determines the default nonce when new accounts are being @@ -38,8 +41,9 @@ var StartingNonce uint64 // * Contracts // * Accounts type StateDB struct { - db ethdb.Database + ca *access.ChainAccess trie *trie.SecureTrie + ctx context.Context stateObjects map[string]*StateObject @@ -51,22 +55,33 @@ type StateDB struct { logSize uint } -// Create a new state from a given trie -func New(root common.Hash, db ethdb.Database) (*StateDB, error) { - tr, err := trie.NewSecure(root, db) +// New creates a new state from a given trie +func New(root common.Hash, ca *access.ChainAccess) (*StateDB, error) { + return NewOdr(access.NoOdr, root, ca) +} + +// NewOdr creates a new state from a given trie with ODR option +func NewOdr(ctx context.Context, root common.Hash, ca *access.ChainAccess) (*StateDB, error) { + tr, err := trie.NewSecureOdr(ctx, root, ca.Db(), requests.NewTrieAccess(ca, root, ca.Db())) if err != nil { glog.Errorf("can't create state trie with root %x: %v", root[:], err) return nil, err } return &StateDB{ - db: db, + ca: ca, trie: tr, + ctx: ctx, stateObjects: make(map[string]*StateObject), refund: new(big.Int), logs: make(map[common.Hash]vm.Logs), }, nil } +// Ctx returns the ODR context of the state +func (self *StateDB) Ctx() context.Context { + return self.ctx +} + func (self *StateDB) StartRecord(thash, bhash common.Hash, ti int) { self.thash = thash self.bhash = bhash @@ -208,7 +223,7 @@ func (self *StateDB) UpdateStateObject(stateObject *StateObject) { //addr := stateObject.Address() if len(stateObject.CodeHash()) > 0 { - self.db.Put(stateObject.CodeHash(), stateObject.code) + self.ca.Db().Put(stateObject.CodeHash(), stateObject.code) } addr := stateObject.Address() self.trie.Update(addr[:], stateObject.RlpEncode()) @@ -220,7 +235,6 @@ func (self *StateDB) DeleteStateObject(stateObject *StateObject) { addr := stateObject.Address() self.trie.Delete(addr[:]) - //delete(self.stateObjects, addr.Str()) } // Retrieve a state object given my the address. Nil if not found @@ -239,7 +253,7 @@ func (self *StateDB) GetStateObject(addr common.Address) (stateObject *StateObje return nil } - stateObject = NewStateObjectFromBytes(addr, []byte(data), self.db) + stateObject = NewStateObjectFromBytes(self.ctx, addr, []byte(data), self.ca) self.SetStateObject(stateObject) return stateObject @@ -265,7 +279,7 @@ func (self *StateDB) newStateObject(addr common.Address) *StateObject { glog.Infof("(+) %x\n", addr) } - stateObject := NewStateObject(addr, self.db) + stateObject := NewStateObject(self.ctx, addr, self.ca) stateObject.SetNonce(StartingNonce) self.stateObjects[addr.Str()] = stateObject @@ -295,12 +309,27 @@ func (self *StateDB) CreateAccount(addr common.Address) vm.Account { // Setting, copying of the state methods // +// CopyWithCtx creates a copy of the state, keeping the original ODR context +func (self *StateDB) CopyWithCtx() *StateDB { + return self.CopyOdr(self.ctx) +} + +// Copy creates a copy of the state with no ODR option func (self *StateDB) Copy() *StateDB { + return self.CopyOdr(access.NoOdr) +} + +// CopyOdr creates a copy of the state with a new ODR context +func (self *StateDB) CopyOdr(ctx context.Context) *StateDB { // ignore error - we assume state-to-be-copied always exists - state, _ := New(common.Hash{}, self.db) - state.trie = self.trie + state, _ := NewOdr(ctx, common.Hash{}, self.ca) + if access.IsOdrContext(ctx) { + state.trie = self.trie.CopySecureWithOdr(ctx, requests.NewTrieAccess(self.ca, common.BytesToHash(self.trie.Root()), self.ca.Db())) + } else { + state.trie = self.trie + } for k, stateObject := range self.stateObjects { - state.stateObjects[k] = stateObject.Copy() + state.stateObjects[k] = stateObject.CopyOdr(ctx) } state.refund.Set(self.refund) @@ -348,14 +377,14 @@ func (s *StateDB) IntermediateRoot() common.Hash { // Commit commits all state changes to the database. func (s *StateDB) Commit() (root common.Hash, err error) { - return s.commit(s.db) + return s.commit(s.ca.Db()) } // CommitBatch commits all state changes to a write batch but does not // execute the batch. It is used to validate state changes against // the root hash stored in a block. func (s *StateDB) CommitBatch() (root common.Hash, batch ethdb.Batch) { - batch = s.db.NewBatch() + batch = s.ca.Db().NewBatch() root, _ = s.commit(batch) return root, batch } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 0dab372ba0..1a175bf3a3 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" ) @@ -38,7 +39,7 @@ type testAccount struct { func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { // Create an empty state db, _ := ethdb.NewMemDatabase() - state, _ := New(common.Hash{}, db) + state, _ := New(common.Hash{}, access.NewDbChainAccess(db)) // Fill it with some arbitrary data accounts := []*testAccount{} @@ -68,7 +69,7 @@ func makeTestState() (ethdb.Database, common.Hash, []*testAccount) { // checkStateAccounts cross references a reconstructed state with an expected // account array. func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) { - state, _ := New(root, db) + state, _ := New(root, access.NewDbChainAccess(db)) for i, acc := range accounts { if balance := state.GetBalance(acc.address); balance.Cmp(acc.balance) != 0 { diff --git a/core/transaction_pool_test.go b/core/transaction_pool_test.go index 229dcacf37..ccf030e3e7 100644 --- a/core/transaction_pool_test.go +++ b/core/transaction_pool_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -36,7 +37,7 @@ func transaction(nonce uint64, gaslimit *big.Int, key *ecdsa.PrivateKey) *types. func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, access.NewDbChainAccess(db)) var m event.TypeMux key, _ := crypto.GenerateKey() @@ -163,7 +164,7 @@ func TestTransactionChainFork(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, access.NewDbChainAccess(db)) pool.currentState = func() (*state.StateDB, error) { return statedb, nil } currentState, _ := pool.currentState() currentState.AddBalance(addr, big.NewInt(100000000000000)) @@ -189,7 +190,7 @@ func TestTransactionDoubleNonce(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, access.NewDbChainAccess(db)) pool.currentState = func() (*state.StateDB, error) { return statedb, nil } currentState, _ := pool.currentState() currentState.AddBalance(addr, big.NewInt(100000000000000)) diff --git a/core/transaction_util.go b/core/transaction_util.go index e2e5b9aeea..fcb9407fcd 100644 --- a/core/transaction_util.go +++ b/core/transaction_util.go @@ -20,17 +20,20 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" + "github.com/ethereum/go-ethereum/core/requests" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/rlp" "github.com/syndtr/goleveldb/leveldb" + "golang.org/x/net/context" ) var ( - receiptsPre = []byte("receipts-") blockReceiptsPre = []byte("receipts-block-") + receiptsPre = []byte("receipts-") ) // PutTransactions stores the transactions in the given database @@ -119,8 +122,8 @@ func DeleteReceipt(db ethdb.Database, txHash common.Hash) { } // GetReceipt returns a receipt by hash -func GetReceipt(db ethdb.Database, txHash common.Hash) *types.Receipt { - data, _ := db.Get(append(receiptsPre, txHash[:]...)) +func GetReceipt(ca *access.ChainAccess, txHash common.Hash) *types.Receipt { + data, _ := ca.Db().Get(append(receiptsPre, txHash[:]...)) if len(data) == 0 { return nil } @@ -134,21 +137,16 @@ func GetReceipt(db ethdb.Database, txHash common.Hash) *types.Receipt { // GetBlockReceipts returns the receipts generated by the transactions // included in block's given hash. -func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts { - data, _ := db.Get(append(blockReceiptsPre, hash[:]...)) - if len(data) == 0 { - return nil - } - rs := []*types.ReceiptForStorage{} - if err := rlp.DecodeBytes(data, &rs); err != nil { - glog.V(logger.Error).Infof("invalid receipt array RLP for hash %x: %v", hash, err) - return nil - } - receipts := make(types.Receipts, len(rs)) - for i, receipt := range rs { - receipts[i] = (*types.Receipt)(receipt) - } - return receipts +func GetBlockReceipts(ca *access.ChainAccess, hash common.Hash) types.Receipts { + return GetBlockReceiptsOdr(access.NoOdr, ca, hash) +} + +// GetBlockReceiptsOdr returns the receipts generated by the transactions +// included in block's given hash from the database or network. +func GetBlockReceiptsOdr(ctx context.Context, ca *access.ChainAccess, hash common.Hash) types.Receipts { + r := requests.NewReceiptsAccess(ca.Db(), hash, GetHeader, PutReceipts, PutBlockReceipts) + ca.Retrieve(ctx, r) + return r.GetReceipts() } // PutBlockReceipts stores the block's transactions associated receipts diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index dd3aa1b0b5..3b11a8d1ea 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -21,6 +21,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -96,7 +97,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { var ( db, _ = ethdb.NewMemDatabase() - statedb, _ = state.New(common.Hash{}, db) + statedb, _ = state.New(common.Hash{}, access.NewDbChainAccess(db)) vmenv = NewEnv(cfg, statedb) sender = statedb.CreateAccount(cfg.Origin) receiver = statedb.CreateAccount(common.StringToAddress("contract")) diff --git a/core/vm_env.go b/core/vm_env.go index c8b50debc6..d1fb9ae41d 100644 --- a/core/vm_env.go +++ b/core/vm_env.go @@ -76,7 +76,7 @@ func (self *VMEnv) CanTransfer(from common.Address, balance *big.Int) bool { } func (self *VMEnv) MakeSnapshot() vm.Database { - return self.state.Copy() + return self.state.CopyWithCtx() } func (self *VMEnv) SetSnapshot(copy vm.Database) { diff --git a/eth/backend.go b/eth/backend.go index 761a17a8ff..387eb2712d 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/common/compiler" "github.com/ethereum/go-ethereum/common/httpclient" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -231,6 +232,8 @@ type Ethereum struct { chainDb ethdb.Database // Block chain database dappDb ethdb.Database // Dapp database + chainAccess *access.ChainAccess // blockchain access layer + //*** SERVICES *** // State manager for processing new blocks and managing the over all states blockProcessor *core.BlockProcessor @@ -297,10 +300,10 @@ func New(config *Config) (*Ethereum, error) { if err := upgradeChainDatabase(chainDb); err != nil { return nil, err } - if err := addMipmapBloomBins(chainDb); err != nil { + chainAccess := access.NewChainAccess(chainDb, false) + if err := addMipmapBloomBins(chainAccess); err != nil { return nil, err } - dappDb, err := newdb(filepath.Join(config.DataDir, "dapp")) if err != nil { if errno, ok := err.(syscall.Errno); ok && datadirInUseErrnos[uint(errno)] { @@ -366,6 +369,7 @@ func New(config *Config) (*Ethereum, error) { eth := &Ethereum{ shutdownChan: make(chan bool), chainDb: chainDb, + chainAccess: chainAccess, dappDb: dappDb, eventMux: &event.TypeMux{}, accountManager: config.AccountManager, @@ -397,7 +401,7 @@ func New(config *Config) (*Ethereum, error) { eth.pow = ethash.New() } //genesis := core.GenesisBlock(uint64(config.GenesisNonce), stateDb) - eth.blockchain, err = core.NewBlockChain(chainDb, eth.pow, eth.EventMux()) + eth.blockchain, err = core.NewBlockChain(chainAccess, eth.pow, eth.EventMux()) if err != nil { if err == core.ErrNoGenesis { return nil, fmt.Errorf(`Genesis block not found. Please supply a genesis block with the "--genesis /path/to/file" argument`) @@ -407,11 +411,13 @@ func New(config *Config) (*Ethereum, error) { newPool := core.NewTxPool(eth.EventMux(), eth.blockchain.State, eth.blockchain.GasLimit) eth.txPool = newPool - eth.blockProcessor = core.NewBlockProcessor(chainDb, eth.pow, eth.blockchain, eth.EventMux()) + eth.blockProcessor = core.NewBlockProcessor(chainAccess, eth.pow, eth.blockchain, eth.EventMux()) eth.blockchain.SetProcessor(eth.blockProcessor) - if eth.protocolManager, err = NewProtocolManager(config.FastSync, config.NetworkId, eth.eventMux, eth.txPool, eth.pow, eth.blockchain, chainDb); err != nil { + + if eth.protocolManager, err = NewProtocolManager(config.FastSync, config.NetworkId, eth.eventMux, eth.txPool, eth.pow, eth.blockchain, chainAccess); err != nil { return nil, err } + eth.miner = miner.New(eth, eth.EventMux(), eth.pow) eth.miner.SetGasPrice(config.GasPrice) eth.miner.SetExtra(config.ExtraData) @@ -493,6 +499,7 @@ func (s *Ethereum) TxPool() *core.TxPool { return s.txPool } func (s *Ethereum) Whisper() *whisper.Whisper { return s.whisper } func (s *Ethereum) EventMux() *event.TypeMux { return s.eventMux } func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } +func (s *Ethereum) ChainAccess() *access.ChainAccess { return s.chainAccess } func (s *Ethereum) DappDb() ethdb.Database { return s.dappDb } func (s *Ethereum) IsListening() bool { return true } // Always listening func (s *Ethereum) PeerCount() int { return s.net.PeerCount() } @@ -732,9 +739,10 @@ func upgradeChainDatabase(db ethdb.Database) error { return nil } -func addMipmapBloomBins(db ethdb.Database) (err error) { +func addMipmapBloomBins(ca *access.ChainAccess) (err error) { const mipmapVersion uint = 2 + db := ca.Db() // check if the version is set. We ignore data for now since there's // only one version so we can easily ignore it for now var data []byte @@ -756,7 +764,7 @@ func addMipmapBloomBins(db ethdb.Database) (err error) { return } }() - latestBlock := core.GetBlock(db, core.GetHeadBlockHash(db)) + latestBlock := core.GetBlock(ca, core.GetHeadBlockHash(db)) if latestBlock == nil { // clean database return } @@ -768,7 +776,7 @@ func addMipmapBloomBins(db ethdb.Database) (err error) { if (hash == common.Hash{}) { return fmt.Errorf("chain db corrupted. Could not find block %d.", i) } - core.WriteMipmapBloom(db, i, core.GetBlockReceipts(db, hash)) + core.WriteMipmapBloom(db, i, core.GetBlockReceipts(ca, hash)) } glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart)) return nil diff --git a/eth/backend_test.go b/eth/backend_test.go index 0379fc843a..fdabf4d64e 100644 --- a/eth/backend_test.go +++ b/eth/backend_test.go @@ -6,6 +6,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" @@ -50,7 +51,7 @@ func TestMipmapUpgrade(t *testing.T) { } } - err := addMipmapBloomBins(db) + err := addMipmapBloomBins(access.NewDbChainAccess(db)) if err != nil { t.Fatal(err) } diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index ef6f74a6ba..b8dd075477 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -682,7 +683,7 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot) } if index > 0 { - if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, tester.stateDb); statedb == nil || err != nil { + if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, access.NewDbChainAccess(tester.stateDb)); statedb == nil || err != nil { t.Fatalf("state reconstruction failed: %v", err) } } diff --git a/eth/filters/filter.go b/eth/filters/filter.go index ff192cdf6b..5bfd1beec2 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" @@ -32,6 +33,7 @@ type AccountChange struct { // Filtering interface type Filter struct { + ca *access.ChainAccess db ethdb.Database begin, end int64 addresses []common.Address @@ -42,10 +44,15 @@ type Filter struct { LogsCallback func(vm.Logs) } -// Create a new filter which uses a bloom filter on blocks to figure out whether a particular block +// New creates a new filter which uses a bloom filter on blocks to figure out whether a particular block // is interesting or not. -func New(db ethdb.Database) *Filter { - return &Filter{db: db} +func New(ca *access.ChainAccess) *Filter { + return &Filter{ca: ca, db: ca.Db()} +} + +// NewWithDb creates a filter with no ODR option +func NewWithDb(db ethdb.Database) *Filter { + return &Filter{ca: access.NewDbChainAccess(db), db: db} } // Set the earliest and latest block for filtering. @@ -69,7 +76,7 @@ func (self *Filter) SetTopics(topics [][]common.Hash) { // Run filters logs with the current parameters set func (self *Filter) Find() vm.Logs { - latestBlock := core.GetBlock(self.db, core.GetHeadBlockHash(self.db)) + latestBlock := core.GetBlock(self.ca, core.GetHeadBlockHash(self.db)) var beginBlockNo uint64 = uint64(self.begin) if self.begin == -1 { beginBlockNo = latestBlock.NumberU64() @@ -124,7 +131,7 @@ func (self *Filter) getLogs(start, end uint64) (logs vm.Logs) { for i := start; i <= end; i++ { hash := core.GetCanonicalHash(self.db, i) if hash != (common.Hash{}) { - block = core.GetBlock(self.db, hash) + block = core.GetBlock(self.ca, hash) } else { // block not found return logs } @@ -134,7 +141,7 @@ func (self *Filter) getLogs(start, end uint64) (logs vm.Logs) { if self.bloomFilter(block) { // Get the logs of the block var ( - receipts = core.GetBlockReceipts(self.db, block.Hash()) + receipts = core.GetBlockReceipts(self.ca, block.Hash()) unfiltered vm.Logs ) for _, receipt := range receipts { @@ -142,6 +149,7 @@ func (self *Filter) getLogs(start, end uint64) (logs vm.Logs) { } logs = append(logs, self.FilterLogs(unfiltered)...) } + block = core.GetBlock(self.ca, block.ParentHash()) } return logs diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index a5418e2e74..22e515aa87 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -84,7 +84,7 @@ func BenchmarkMipmaps(b *testing.B) { } b.ResetTimer() - filter := New(db) + filter := NewWithDb(db) filter.SetAddresses([]common.Address{addr1, addr2, addr3, addr4}) filter.SetBeginBlock(0) filter.SetEndBlock(-1) @@ -185,7 +185,7 @@ func TestFilters(t *testing.T) { } } - filter := New(db) + filter := NewWithDb(db) filter.SetAddresses([]common.Address{addr}) filter.SetTopics([][]common.Hash{[]common.Hash{hash1, hash2, hash3, hash4}}) filter.SetBeginBlock(0) @@ -196,7 +196,7 @@ func TestFilters(t *testing.T) { t.Error("expected 4 log, got", len(logs)) } - filter = New(db) + filter = NewWithDb(db) filter.SetAddresses([]common.Address{addr}) filter.SetTopics([][]common.Hash{[]common.Hash{hash3}}) filter.SetBeginBlock(900) @@ -209,7 +209,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = New(db) + filter = NewWithDb(db) filter.SetAddresses([]common.Address{addr}) filter.SetTopics([][]common.Hash{[]common.Hash{hash3}}) filter.SetBeginBlock(990) @@ -222,7 +222,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = New(db) + filter = NewWithDb(db) filter.SetTopics([][]common.Hash{[]common.Hash{hash1, hash2}}) filter.SetBeginBlock(1) filter.SetEndBlock(10) @@ -233,7 +233,7 @@ func TestFilters(t *testing.T) { } failHash := common.BytesToHash([]byte("fail")) - filter = New(db) + filter = NewWithDb(db) filter.SetTopics([][]common.Hash{[]common.Hash{failHash}}) filter.SetBeginBlock(0) filter.SetEndBlock(-1) @@ -244,7 +244,7 @@ func TestFilters(t *testing.T) { } failAddr := common.BytesToAddress([]byte("failmenow")) - filter = New(db) + filter = NewWithDb(db) filter.SetAddresses([]common.Address{failAddr}) filter.SetBeginBlock(0) filter.SetEndBlock(-1) @@ -254,7 +254,7 @@ func TestFilters(t *testing.T) { t.Error("expected 0 log, got", len(logs)) } - filter = New(db) + filter = NewWithDb(db) filter.SetTopics([][]common.Hash{[]common.Hash{failHash}, []common.Hash{hash1}}) filter.SetBeginBlock(0) filter.SetEndBlock(-1) diff --git a/eth/handler.go b/eth/handler.go index d8c5b4b648..c2e4ae17ab 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -26,10 +26,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/fetcher" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -58,10 +58,10 @@ type blockFetcherFn func([]common.Hash) error type ProtocolManager struct { networkId int - fastSync bool - txpool txPool - blockchain *core.BlockChain - chaindb ethdb.Database + fastSync bool + txpool txPool + blockchain *core.BlockChain + chainAccess *access.ChainAccess downloader *downloader.Downloader fetcher *fetcher.Fetcher @@ -86,7 +86,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(fastSync bool, networkId int, mux *event.TypeMux, txpool txPool, pow pow.PoW, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) { +func NewProtocolManager(fastSync bool, networkId int, mux *event.TypeMux, txpool txPool, pow pow.PoW, blockchain *core.BlockChain, ca *access.ChainAccess) (*ProtocolManager, error) { // Figure out whether to allow fast sync or not if fastSync && blockchain.CurrentBlock().NumberU64() > 0 { glog.V(logger.Info).Infof("blockchain not empty, fast sync disabled") @@ -94,22 +94,22 @@ func NewProtocolManager(fastSync bool, networkId int, mux *event.TypeMux, txpool } // Create the protocol manager with the base fields manager := &ProtocolManager{ - networkId: networkId, - fastSync: fastSync, - eventMux: mux, - txpool: txpool, - blockchain: blockchain, - chaindb: chaindb, - peers: newPeerSet(), - newPeerCh: make(chan *peer, 1), - txsyncCh: make(chan *txsync), - quitSync: make(chan struct{}), + networkId: networkId, + fastSync: fastSync, + eventMux: mux, + txpool: txpool, + blockchain: blockchain, + chainAccess: ca, + peers: newPeerSet(), + newPeerCh: make(chan *peer, 1), + txsyncCh: make(chan *txsync), + quitSync: make(chan struct{}), } // Initiate a sub-protocol for every implemented version we can handle manager.SubProtocols = make([]p2p.Protocol, 0, len(ProtocolVersions)) for i, version := range ProtocolVersions { // Skip protocol version if incompatible with the mode of operation - if fastSync && version < eth63 { + if manager.fastSync && version < eth63 { continue } // Compatible; initialise the sub-protocol @@ -137,9 +137,8 @@ func NewProtocolManager(fastSync bool, networkId int, mux *event.TypeMux, txpool if len(manager.SubProtocols) == 0 { return nil, errIncompatibleConfig } - // Construct the different synchronisation mechanisms - manager.downloader = downloader.New(chaindb, manager.eventMux, blockchain.HasHeader, blockchain.HasBlock, blockchain.GetHeader, blockchain.GetBlock, - blockchain.CurrentHeader, blockchain.CurrentBlock, blockchain.CurrentFastBlock, blockchain.FastSyncCommitHead, blockchain.GetTd, + manager.downloader = downloader.New(ca.Db(), manager.eventMux, blockchain.HasHeader, blockchain.HasBlock, blockchain.GetHeader, + blockchain.GetBlock, blockchain.CurrentHeader, blockchain.CurrentBlock, blockchain.CurrentFastBlock, blockchain.FastSyncCommitHead, blockchain.GetTd, blockchain.InsertHeaderChain, blockchain.InsertChain, blockchain.InsertReceiptChain, blockchain.Rollback, manager.removePeer) validator := func(block *types.Block, parent *types.Block) error { @@ -233,6 +232,7 @@ func (pm *ProtocolManager) handle(p *peer) error { p.RequestHeadersByNumber, p.RequestBodies, p.RequestReceipts, p.RequestNodeData); err != nil { return err } + // Propagate existing transactions. new transactions appearing // after this will be sent via broadcasts. pm.syncTransactions(p) @@ -292,7 +292,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { request.Amount = uint64(downloader.MaxHashFetch) } // Calculate the last block that should be retrieved, and short circuit if unavailable - last := pm.blockchain.GetBlockByNumber(request.Number + request.Amount - 1) + last := pm.blockchain.GetBlockByNumber(request.Number+request.Amount-1) if last == nil { last = pm.blockchain.CurrentBlock() request.Amount = last.NumberU64() - request.Number + 1 @@ -517,7 +517,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } // Retrieve the requested state entry, stopping if enough was found - if entry, err := pm.chaindb.Get(hash.Bytes()); err == nil { + if entry, err := pm.chainAccess.Db().Get(hash.Bytes()); err == nil { data = append(data, entry) bytes += len(entry) } @@ -555,7 +555,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } // Retrieve the requested block's receipts, skipping if unknown to us - results := core.GetBlockReceipts(pm.chaindb, hash) + results := core.GetBlockReceipts(pm.chainAccess, hash) if results == nil { if header := pm.blockchain.GetHeader(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { continue diff --git a/eth/handler_test.go b/eth/handler_test.go index ab2ce54b13..a9a4a22a74 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -35,7 +36,6 @@ func TestProtocolCompatibility(t *testing.T) { // Try all available compatibility configs and check for errors for i, tt := range tests { ProtocolVersions = []uint{tt.version} - pm, err := newTestProtocolManager(tt.fastSync, 0, nil, nil) if pm != nil { defer pm.Stop() @@ -62,14 +62,14 @@ func testGetBlockHashes(t *testing.T, protocol int) { number int result int }{ - {common.Hash{}, 1, 0}, // Make sure non existent hashes don't return results - {pm.blockchain.Genesis().Hash(), 1, 0}, // There are no hashes to retrieve up from the genesis - {pm.blockchain.GetBlockByNumber(5).Hash(), 5, 5}, // All the hashes including the genesis requested - {pm.blockchain.GetBlockByNumber(5).Hash(), 10, 5}, // More hashes than available till the genesis requested - {pm.blockchain.GetBlockByNumber(100).Hash(), 10, 10}, // All hashes available from the middle of the chain - {pm.blockchain.CurrentBlock().Hash(), 10, 10}, // All hashes available from the head of the chain - {pm.blockchain.CurrentBlock().Hash(), limit, limit}, // Request the maximum allowed hash count - {pm.blockchain.CurrentBlock().Hash(), limit + 1, limit}, // Request more than the maximum allowed hash count + {common.Hash{}, 1, 0}, // Make sure non existent hashes don't return results + {pm.blockchain.Genesis().Hash(), 1, 0}, // There are no hashes to retrieve up from the genesis + {pm.blockchain.GetBlockByNumber(5).Hash(), 5, 5}, // All the hashes including the genesis requested + {pm.blockchain.GetBlockByNumber(5).Hash(), 10, 5}, // More hashes than available till the genesis requested + {pm.blockchain.GetBlockByNumber(100).Hash(), 10, 10}, // All hashes available from the middle of the chain + {pm.blockchain.CurrentBlock().Hash(), 10, 10}, // All hashes available from the head of the chain + {pm.blockchain.CurrentBlock().Hash(), limit, limit}, // Request the maximum allowed hash count + {pm.blockchain.CurrentBlock().Hash(), limit + 1, limit}, // Request more than the maximum allowed hash count } // Run each of the tests and verify the results against the chain for i, tt := range tests { @@ -78,7 +78,7 @@ func testGetBlockHashes(t *testing.T, protocol int) { if len(resp) > 0 { from := pm.blockchain.GetBlock(tt.origin).NumberU64() - 1 for j := 0; j < len(resp); j++ { - resp[j] = pm.blockchain.GetBlockByNumber(uint64(int(from) - j)).Hash() + resp[j] = pm.blockchain.GetBlockByNumber(uint64(int(from)-j)).Hash() } } // Send the hash request and verify the response @@ -120,7 +120,7 @@ func testGetBlockHashesFromNumber(t *testing.T, protocol int) { // Assemble the hash response we would like to receive resp := make([]common.Hash, tt.result) for j := 0; j < len(resp); j++ { - resp[j] = pm.blockchain.GetBlockByNumber(tt.origin + uint64(j)).Hash() + resp[j] = pm.blockchain.GetBlockByNumber(tt.origin+uint64(j)).Hash() } // Send the hash request and verify the response p2p.Send(peer.app, 0x08, getBlockHashesFromNumberData{tt.origin, uint64(tt.number)}) @@ -222,42 +222,42 @@ func testGetBlockHeaders(t *testing.T, protocol int) { }{ // A single random block should be retrievable by hash and number too { - &getBlockHeadersData{Origin: hashOrNumber{Hash: pm.blockchain.GetBlockByNumber(limit / 2).Hash()}, Amount: 1}, - []common.Hash{pm.blockchain.GetBlockByNumber(limit / 2).Hash()}, + &getBlockHeadersData{Origin: hashOrNumber{Hash: pm.blockchain.GetBlockByNumber(limit/2).Hash()}, Amount: 1}, + []common.Hash{pm.blockchain.GetBlockByNumber(limit/2).Hash()}, }, { &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 1}, - []common.Hash{pm.blockchain.GetBlockByNumber(limit / 2).Hash()}, + []common.Hash{pm.blockchain.GetBlockByNumber(limit/2).Hash()}, }, // Multiple headers should be retrievable in both directions { &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 3}, []common.Hash{ - pm.blockchain.GetBlockByNumber(limit / 2).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 + 1).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 + 2).Hash(), + pm.blockchain.GetBlockByNumber(limit/2).Hash(), + pm.blockchain.GetBlockByNumber(limit/2+1).Hash(), + pm.blockchain.GetBlockByNumber(limit/2+2).Hash(), }, }, { &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Amount: 3, Reverse: true}, []common.Hash{ - pm.blockchain.GetBlockByNumber(limit / 2).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 - 1).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 - 2).Hash(), + pm.blockchain.GetBlockByNumber(limit/2).Hash(), + pm.blockchain.GetBlockByNumber(limit/2-1).Hash(), + pm.blockchain.GetBlockByNumber(limit/2-2).Hash(), }, }, // Multiple headers with skip lists should be retrievable { &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Skip: 3, Amount: 3}, []common.Hash{ - pm.blockchain.GetBlockByNumber(limit / 2).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 + 4).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 + 8).Hash(), + pm.blockchain.GetBlockByNumber(limit/2).Hash(), + pm.blockchain.GetBlockByNumber(limit/2+4).Hash(), + pm.blockchain.GetBlockByNumber(limit/2+8).Hash(), }, }, { &getBlockHeadersData{Origin: hashOrNumber{Number: limit / 2}, Skip: 3, Amount: 3, Reverse: true}, []common.Hash{ - pm.blockchain.GetBlockByNumber(limit / 2).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 - 4).Hash(), - pm.blockchain.GetBlockByNumber(limit/2 - 8).Hash(), + pm.blockchain.GetBlockByNumber(limit/2).Hash(), + pm.blockchain.GetBlockByNumber(limit/2-4).Hash(), + pm.blockchain.GetBlockByNumber(limit/2-8).Hash(), }, }, // The chain endpoints should be retrievable @@ -277,7 +277,7 @@ func testGetBlockHeaders(t *testing.T, protocol int) { { &getBlockHeadersData{Origin: hashOrNumber{Number: pm.blockchain.CurrentBlock().NumberU64() - 4}, Skip: 3, Amount: 3}, []common.Hash{ - pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64() - 4).Hash(), + pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64()-4).Hash(), pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64()).Hash(), }, }, { @@ -291,8 +291,8 @@ func testGetBlockHeaders(t *testing.T, protocol int) { { &getBlockHeadersData{Origin: hashOrNumber{Number: pm.blockchain.CurrentBlock().NumberU64() - 4}, Skip: 2, Amount: 3}, []common.Hash{ - pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64() - 4).Hash(), - pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64() - 1).Hash(), + pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64()-4).Hash(), + pm.blockchain.GetBlockByNumber(pm.blockchain.CurrentBlock().NumberU64()-1).Hash(), }, }, { &getBlockHeadersData{Origin: hashOrNumber{Number: 4}, Skip: 2, Amount: 3, Reverse: true}, @@ -365,7 +365,7 @@ func testGetBlockBodies(t *testing.T, protocol int) { for i, tt := range tests { // Collect the hashes to request, and the response to expect hashes, seen := []common.Hash{}, make(map[int64]bool) - bodies := []*blockBody{} + bodies := []*types.Body{} for j := 0; j < tt.random; j++ { for { @@ -376,7 +376,7 @@ func testGetBlockBodies(t *testing.T, protocol int) { block := pm.blockchain.GetBlockByNumber(uint64(num)) hashes = append(hashes, block.Hash()) if len(bodies) < tt.expected { - bodies = append(bodies, &blockBody{Transactions: block.Transactions(), Uncles: block.Uncles()}) + bodies = append(bodies, &types.Body{Transactions: block.Transactions(), Uncles: block.Uncles()}) } break } @@ -386,7 +386,7 @@ func testGetBlockBodies(t *testing.T, protocol int) { hashes = append(hashes, hash) if tt.available[j] && len(bodies) < tt.expected { block := pm.blockchain.GetBlock(hash) - bodies = append(bodies, &blockBody{Transactions: block.Transactions(), Uncles: block.Uncles()}) + bodies = append(bodies, &types.Body{Transactions: block.Transactions(), Uncles: block.Uncles()}) } } // Send the hash request and verify the response @@ -442,7 +442,7 @@ func testGetNodeData(t *testing.T, protocol int) { // Fetch for now the entire chain db hashes := []common.Hash{} - for _, key := range pm.chaindb.(*ethdb.MemDatabase).Keys() { + for _, key := range pm.chainAccess.Db().(*ethdb.MemDatabase).Keys() { if len(key) == len(common.Hash{}) { hashes = append(hashes, common.BytesToHash(key)) } @@ -471,7 +471,7 @@ func testGetNodeData(t *testing.T, protocol int) { } accounts := []common.Address{testBankAddress, acc1Addr, acc2Addr} for i := uint64(0); i <= pm.blockchain.CurrentBlock().NumberU64(); i++ { - trie, _ := state.New(pm.blockchain.GetBlockByNumber(i).Root(), statedb) + trie, _ := state.New(pm.blockchain.GetBlockByNumber(i).Root(), access.NewDbChainAccess(statedb)) for j, acc := range accounts { state, _ := pm.blockchain.State() @@ -537,7 +537,7 @@ func testGetReceipt(t *testing.T, protocol int) { block := pm.blockchain.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) - receipts = append(receipts, core.GetBlockReceipts(pm.chaindb, block.Hash())) + receipts = append(receipts, core.GetBlockReceipts(pm.chainAccess, block.Hash())) } // Send the hash request and verify the response p2p.Send(peer.app, 0x0f, hashes) diff --git a/eth/helper_test.go b/eth/helper_test.go index 65fccf7b4d..875cc50217 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" @@ -33,16 +34,17 @@ func newTestProtocolManager(fastSync bool, blocks int, generator func(int, *core evmux = new(event.TypeMux) pow = new(core.FakePow) db, _ = ethdb.NewMemDatabase() + ca = access.NewDbChainAccess(db) genesis = core.WriteGenesisBlockForTesting(db, core.GenesisAccount{testBankAddress, testBankFunds}) - blockchain, _ = core.NewBlockChain(db, pow, evmux) - blockproc = core.NewBlockProcessor(db, pow, blockchain, evmux) + blockchain, _ = core.NewBlockChain(ca, pow, evmux) + blockproc = core.NewBlockProcessor(ca, pow, blockchain, evmux) ) blockchain.SetProcessor(blockproc) chain, _ := core.GenerateChain(genesis, db, blocks, generator) if _, err := blockchain.InsertChain(chain); err != nil { panic(err) } - pm, err := NewProtocolManager(fastSync, NetworkId, evmux, &testTxPool{added: newtx}, pow, blockchain, db) + pm, err := NewProtocolManager(fastSync, NetworkId, evmux, &testTxPool{added: newtx}, pow, blockchain, ca) if err != nil { return nil, err } diff --git a/eth/peer.go b/eth/peer.go index 15ba22ff53..0e20d084c7 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -196,7 +196,7 @@ func (p *peer) SendBlockHeaders(headers []*types.Header) error { } // SendBlockBodies sends a batch of block contents to the remote peer. -func (p *peer) SendBlockBodies(bodies []*blockBody) error { +func (p *peer) SendBlockBodies(bodies []*types.Body) error { return p2p.Send(p.rw, BlockBodiesMsg, blockBodiesData(bodies)) } diff --git a/eth/protocol.go b/eth/protocol.go index 808ac06011..0d03c12ed6 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -201,14 +201,8 @@ type newBlockData struct { TD *big.Int } -// blockBody represents the data content of a single block. -type blockBody struct { - Transactions []*types.Transaction // Transactions contained within a block - Uncles []*types.Header // Uncles contained within a block -} - // blockBodiesData is the network packet for block content distribution. -type blockBodiesData []*blockBody +type blockBodiesData []*types.Body // nodeDataData is the network response packet for a node data retrieval. type nodeDataData []struct { diff --git a/miner/worker.go b/miner/worker.go index 2d072ef60b..99ba7719f5 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -359,7 +359,7 @@ func (self *worker) push(work *Work) { // makeCurrent creates a new environment for the current cycle. func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error { - state, err := state.New(parent.Root(), self.eth.ChainDb()) + state, err := state.New(parent.Root(), self.eth.ChainAccess()) if err != nil { return err } diff --git a/rpc/api/debug.go b/rpc/api/debug.go index d2cbc7f19d..b48ef54634 100644 --- a/rpc/api/debug.go +++ b/rpc/api/debug.go @@ -119,7 +119,7 @@ func (self *debugApi) DumpBlock(req *shared.Request) (interface{}, error) { return nil, fmt.Errorf("block #%d not found", args.BlockNumber) } - stateDb, err := state.New(block.Root(), self.ethereum.ChainDb()) + stateDb, err := state.New(block.Root(), self.ethereum.ChainAccess()) if err != nil { return nil, err } diff --git a/rpc/api/eth.go b/rpc/api/eth.go index b84ae31da2..da75fa1525 100644 --- a/rpc/api/eth.go +++ b/rpc/api/eth.go @@ -19,9 +19,8 @@ package api import ( "bytes" "encoding/json" - "math/big" - "fmt" + "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/natspec" @@ -29,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/rpc/codec" "github.com/ethereum/go-ethereum/rpc/shared" "github.com/ethereum/go-ethereum/xeth" + "golang.org/x/net/context" "gopkg.in/fatih/set.v0" ) @@ -153,8 +153,8 @@ func (self *ethApi) GetBalance(req *shared.Request) (interface{}, error) { if err := self.codec.Decode(req.Params, &args); err != nil { return nil, shared.NewDecodeParamError(err.Error()) } - - return self.xeth.AtStateNum(args.BlockNumber).BalanceAt(args.Address), nil + res := self.xeth.WithCtx(req.GetCtx()).AtStateNum(args.BlockNumber).BalanceAt(args.Address) + return res, nil } func (self *ethApi) ProtocolVersion(req *shared.Request) (interface{}, error) { @@ -170,6 +170,9 @@ func (self *ethApi) IsMining(req *shared.Request) (interface{}, error) { } func (self *ethApi) IsSyncing(req *shared.Request) (interface{}, error) { + if self.ethereum.Downloader() == nil { + return false, nil + } origin, current, height := self.ethereum.Downloader().Progress() if current < height { return map[string]interface{}{ @@ -191,7 +194,7 @@ func (self *ethApi) GetStorage(req *shared.Request) (interface{}, error) { return nil, shared.NewDecodeParamError(err.Error()) } - return self.xeth.AtStateNum(args.BlockNumber).State().SafeGet(args.Address).Storage(), nil + return self.xeth.WithCtx(req.GetCtx()).AtStateNum(args.BlockNumber).State().SafeGet(args.Address).Storage(), nil } func (self *ethApi) GetStorageAt(req *shared.Request) (interface{}, error) { @@ -200,7 +203,7 @@ func (self *ethApi) GetStorageAt(req *shared.Request) (interface{}, error) { return nil, shared.NewDecodeParamError(err.Error()) } - return self.xeth.AtStateNum(args.BlockNumber).StorageAt(args.Address, args.Key), nil + return self.xeth.WithCtx(req.GetCtx()).AtStateNum(args.BlockNumber).StorageAt(args.Address, args.Key), nil } func (self *ethApi) GetTransactionCount(req *shared.Request) (interface{}, error) { @@ -209,7 +212,7 @@ func (self *ethApi) GetTransactionCount(req *shared.Request) (interface{}, error return nil, shared.NewDecodeParamError(err.Error()) } - count := self.xeth.AtStateNum(args.BlockNumber).TxCountAt(args.Address) + count := self.xeth.WithCtx(req.GetCtx()).AtStateNum(args.BlockNumber).TxCountAt(args.Address) return fmt.Sprintf("%#x", count), nil } @@ -218,7 +221,7 @@ func (self *ethApi) GetBlockTransactionCountByHash(req *shared.Request) (interfa if err := self.codec.Decode(req.Params, &args); err != nil { return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByHash(args.Hash) + block := self.xeth.WithCtx(req.GetCtx()).EthBlockByHash(args.Hash) if block == nil { return nil, nil } @@ -231,7 +234,7 @@ func (self *ethApi) GetBlockTransactionCountByNumber(req *shared.Request) (inter return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByNumber(args.BlockNumber) + block := self.xeth.WithCtx(req.GetCtx()).EthBlockByNumber(args.BlockNumber) if block == nil { return nil, nil } @@ -244,7 +247,7 @@ func (self *ethApi) GetUncleCountByBlockHash(req *shared.Request) (interface{}, return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByHash(args.Hash) + block := self.xeth.WithCtx(req.GetCtx()).EthBlockByHash(args.Hash) if block == nil { return nil, nil } @@ -257,7 +260,7 @@ func (self *ethApi) GetUncleCountByBlockNumber(req *shared.Request) (interface{} return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByNumber(args.BlockNumber) + block := self.xeth.WithCtx(req.GetCtx()).EthBlockByNumber(args.BlockNumber) if block == nil { return nil, nil } @@ -269,7 +272,7 @@ func (self *ethApi) GetData(req *shared.Request) (interface{}, error) { if err := self.codec.Decode(req.Params, &args); err != nil { return nil, shared.NewDecodeParamError(err.Error()) } - v := self.xeth.AtStateNum(args.BlockNumber).CodeAtBytes(args.Address) + v := self.xeth.WithCtx(req.GetCtx()).AtStateNum(args.BlockNumber).CodeAtBytes(args.Address) return newHexData(v), nil } @@ -337,7 +340,7 @@ func (self *ethApi) GetNatSpec(req *shared.Request) (interface{}, error) { } func (self *ethApi) EstimateGas(req *shared.Request) (interface{}, error) { - _, gas, err := self.doCall(req.Params) + _, gas, err := self.doCall(req.GetCtx(), req.Params) if err != nil { return nil, err } @@ -351,7 +354,7 @@ func (self *ethApi) EstimateGas(req *shared.Request) (interface{}, error) { } func (self *ethApi) Call(req *shared.Request) (interface{}, error) { - v, _, err := self.doCall(req.Params) + v, _, err := self.doCall(req.GetCtx(), req.Params) if err != nil { return nil, err } @@ -368,13 +371,13 @@ func (self *ethApi) Flush(req *shared.Request) (interface{}, error) { return nil, shared.NewNotImplementedError(req.Method) } -func (self *ethApi) doCall(params json.RawMessage) (string, string, error) { +func (self *ethApi) doCall(ctx context.Context, params json.RawMessage) (string, string, error) { args := new(CallArgs) if err := self.codec.Decode(params, &args); err != nil { return "", "", err } - return self.xeth.AtStateNum(args.BlockNumber).Call(args.From, args.To, args.Value.String(), args.Gas.String(), args.GasPrice.String(), args.Data) + return self.xeth.WithCtx(ctx).AtStateNum(args.BlockNumber).Call(args.From, args.To, args.Value.String(), args.Gas.String(), args.GasPrice.String(), args.Data) } func (self *ethApi) GetBlockByHash(req *shared.Request) (interface{}, error) { @@ -382,7 +385,7 @@ func (self *ethApi) GetBlockByHash(req *shared.Request) (interface{}, error) { if err := self.codec.Decode(req.Params, &args); err != nil { return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByHash(args.BlockHash) + block := self.xeth.WithCtx(req.GetCtx()).EthBlockByHash(args.BlockHash) if block == nil { return nil, nil } @@ -395,7 +398,7 @@ func (self *ethApi) GetBlockByNumber(req *shared.Request) (interface{}, error) { return nil, shared.NewDecodeParamError(err.Error()) } - block := self.xeth.EthBlockByNumber(args.BlockNumber) + block := self.xeth.WithCtx(req.GetCtx()).EthBlockByNumber(args.BlockNumber) if block == nil { return nil, nil } @@ -408,7 +411,7 @@ func (self *ethApi) GetTransactionByHash(req *shared.Request) (interface{}, erro return nil, shared.NewDecodeParamError(err.Error()) } - tx, bhash, bnum, txi := self.xeth.EthTransactionByHash(args.Hash) + tx, bhash, bnum, txi := self.xeth.EthTransactionByHash(args.Hash) //ODR if tx != nil { v := NewTransactionRes(tx) // if the blockhash is 0, assume this is a pending transaction @@ -428,7 +431,7 @@ func (self *ethApi) GetTransactionByBlockHashAndIndex(req *shared.Request) (inte return nil, shared.NewDecodeParamError(err.Error()) } - raw := self.xeth.EthBlockByHash(args.Hash) + raw := self.xeth.WithCtx(req.GetCtx()).EthBlockByHash(args.Hash) if raw == nil { return nil, nil } @@ -446,7 +449,7 @@ func (self *ethApi) GetTransactionByBlockNumberAndIndex(req *shared.Request) (in return nil, shared.NewDecodeParamError(err.Error()) } - raw := self.xeth.EthBlockByNumber(args.BlockNumber) + raw := self.xeth.WithCtx(req.GetCtx()).EthBlockByNumber(args.BlockNumber) if raw == nil { return nil, nil } @@ -464,7 +467,7 @@ func (self *ethApi) GetUncleByBlockHashAndIndex(req *shared.Request) (interface{ return nil, shared.NewDecodeParamError(err.Error()) } - raw := self.xeth.EthBlockByHash(args.Hash) + raw := self.xeth.WithCtx(req.GetCtx()).EthBlockByHash(args.Hash) if raw == nil { return nil, nil } @@ -482,7 +485,7 @@ func (self *ethApi) GetUncleByBlockNumberAndIndex(req *shared.Request) (interfac return nil, shared.NewDecodeParamError(err.Error()) } - raw := self.xeth.EthBlockByNumber(args.BlockNumber) + raw := self.xeth.WithCtx(req.GetCtx()).EthBlockByNumber(args.BlockNumber) if raw == nil { return nil, nil } @@ -662,7 +665,7 @@ func (self *ethApi) GetTransactionReceipt(req *shared.Request) (interface{}, err txhash := common.BytesToHash(common.FromHex(args.Hash)) tx, bhash, bnum, txi := self.xeth.EthTransactionByHash(args.Hash) - rec := self.xeth.GetTxReceipt(txhash) + rec := self.xeth.GetTxReceipt(txhash) //ODR // We could have an error of "not found". Should disambiguate // if err != nil { // return err, nil diff --git a/rpc/comms/comms.go b/rpc/comms/comms.go index 61fba5722c..b21be624cc 100644 --- a/rpc/comms/comms.go +++ b/rpc/comms/comms.go @@ -19,12 +19,12 @@ package comms import ( "io" "net" - "fmt" "strings" - "strconv" + "time" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/rpc/codec" @@ -69,6 +69,8 @@ func handle(id int, conn net.Conn, api shared.EthereumApi, c codec.Codec) { codec.Close() }() + channelID := access.NewChannelID(time.Second) + for { requests, isBatch, err := codec.ReadRequest() if err == io.EOF { @@ -78,11 +80,21 @@ func handle(id int, conn net.Conn, api shared.EthereumApi, c codec.Codec) { return } + ctx, _ := access.NewContext(channelID) + if isBatch { responses := make([]*interface{}, len(requests)) responseCount := 0 + var res interface{} + var err error for _, req := range requests { - res, err := api.Execute(req) + if access.Terminated(ctx) { + res = nil + err = ctx.Err() + } else { + req.SetCtx(ctx) + res, err = api.Execute(req) + } if req.Id != nil { rpcResponse := shared.NewRpcResponse(req.Id, req.Jsonrpc, res, err) responses[responseCount] = rpcResponse @@ -97,7 +109,12 @@ func handle(id int, conn net.Conn, api shared.EthereumApi, c codec.Codec) { } } else { var rpcResponse interface{} + requests[0].SetCtx(ctx) res, err := api.Execute(requests[0]) + if access.Terminated(ctx) { + res = nil + err = ctx.Err() + } rpcResponse = shared.NewRpcResponse(requests[0].Id, requests[0].Jsonrpc, res, err) err = codec.WriteResponse(rpcResponse) diff --git a/rpc/comms/http.go b/rpc/comms/http.go index f4a930d0ef..67b3653c11 100644 --- a/rpc/comms/http.go +++ b/rpc/comms/http.go @@ -29,6 +29,7 @@ import ( "io" "io/ioutil" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/rpc/codec" @@ -67,13 +68,14 @@ type stopServer struct { type handler struct { codec codec.Codec api shared.EthereumApi + channelID *access.OdrChannelID } // StartHTTP starts listening for RPC requests sent via HTTP. func StartHttp(cfg HttpConfig, codec codec.Codec, api shared.EthereumApi) error { httpServerMu.Lock() defer httpServerMu.Unlock() - + addr := fmt.Sprintf("%s:%d", cfg.ListenAddress, cfg.ListenPort) if httpServer != nil { if addr != httpServer.Addr { @@ -82,7 +84,8 @@ func StartHttp(cfg HttpConfig, codec codec.Codec, api shared.EthereumApi) error return nil // RPC service already running on given host/port } // Set up the request handler, wrapping it with CORS headers if configured. - handler := http.Handler(&handler{codec, api}) + channelID := access.NewChannelID(time.Second) + handler := http.Handler(&handler{codec, api, channelID}) if len(cfg.CorsDomain) > 0 { opts := cors.Options{ AllowedMethods: []string{"POST"}, @@ -121,9 +124,15 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { } c := h.codec.New(nil) + ctx, _ := access.NewContext(h.channelID) var rpcReq shared.Request if err = c.Decode(payload, &rpcReq); err == nil { + rpcReq.SetCtx(ctx) reply, err := h.api.Execute(&rpcReq) + if access.Terminated(ctx) { + reply = nil + err = ctx.Err() + } res := shared.NewRpcResponse(rpcReq.Id, rpcReq.Jsonrpc, reply, err) sendJSON(w, &res) return @@ -133,8 +142,16 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { if err = c.Decode(payload, &reqBatch); err == nil { resBatch := make([]*interface{}, len(reqBatch)) resCount := 0 + var reply interface{} + var err error for i, rpcReq := range reqBatch { - reply, err := h.api.Execute(&rpcReq) + if access.Terminated(ctx) { + reply = nil + err = ctx.Err() + } else { + rpcReq.SetCtx(ctx) + reply, err = h.api.Execute(&rpcReq) + } if rpcReq.Id != nil { // this leaves nil entries in the response batch for later removal resBatch[i] = shared.NewRpcResponse(rpcReq.Id, rpcReq.Jsonrpc, reply, err) resCount += 1 diff --git a/rpc/comms/inproc.go b/rpc/comms/inproc.go index e8058e32bf..aa2b9074d0 100644 --- a/rpc/comms/inproc.go +++ b/rpc/comms/inproc.go @@ -18,7 +18,9 @@ package comms import ( "fmt" + "time" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/rpc/codec" "github.com/ethereum/go-ethereum/rpc/shared" ) @@ -30,12 +32,14 @@ type InProcClient struct { lastJsonrpc string lastErr error lastRes interface{} + channelID *access.OdrChannelID } // Create a new in process client func NewInProcClient(codec codec.Codec) *InProcClient { return &InProcClient{ codec: codec, + channelID: access.NewChannelID(time.Second), } } @@ -52,7 +56,13 @@ func (self *InProcClient) Send(req interface{}) error { if r, ok := req.(*shared.Request); ok { self.lastId = r.Id self.lastJsonrpc = r.Jsonrpc + ctx, _ := access.NewContext(self.channelID) + r.SetCtx(ctx) self.lastRes, self.lastErr = self.api.Execute(r) + if access.Terminated(ctx) { + self.lastRes = nil + self.lastErr = ctx.Err() + } return self.lastErr } diff --git a/rpc/shared/types.go b/rpc/shared/types.go index db328234d2..64eae1a846 100644 --- a/rpc/shared/types.go +++ b/rpc/shared/types.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" + "golang.org/x/net/context" ) // Ethereum RPC API interface @@ -39,11 +40,14 @@ type EthereumApi interface { } // RPC request +// (ODR context is propagated inside the request struct because every +// request has its own context. It does not influence JSON encoding.) type Request struct { Id interface{} `json:"id"` Jsonrpc string `json:"jsonrpc"` Method string `json:"method"` Params json.RawMessage `json:"params"` + ctx context.Context } // RPC response @@ -73,6 +77,16 @@ type ErrorObject struct { // Data interface{} `json:"data"` } +// GetCtx returns the ODR context assigned to the request +func (req *Request) GetCtx() context.Context { + return req.ctx +} + +// SetCtx assigns an ODR context to the request +func (req *Request) SetCtx(ctx context.Context) { + req.ctx = ctx +} + // Create RPC error response, this allows for custom error codes func NewRpcErrorResponse(id interface{}, jsonrpcver string, errCode int, err error) *ErrorResponse { jsonerr := &ErrorObject{errCode, err.Error()} diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 6a2eb96a42..8017eea082 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -217,7 +218,7 @@ func runBlockTest(test *BlockTest) error { // InsertPreState populates the given database with the genesis // accounts defined by the test. func (t *BlockTest) InsertPreState(db ethdb.Database, am *accounts.Manager) (*state.StateDB, error) { - statedb, err := state.New(common.Hash{}, db) + statedb, err := state.New(common.Hash{}, access.NewDbChainAccess(db)) if err != nil { return nil, err } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 352fe3570b..d2016f3e37 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -103,7 +104,7 @@ func BenchStateTest(p string, conf bconf, b *testing.B) error { func benchStateTest(test VmTest, env map[string]string, b *testing.B) { b.StopTimer() db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, access.NewDbChainAccess(db)) for addr, account := range test.Pre { obj := StateObjectFromAccount(db, addr, account) statedb.SetStateObject(obj) @@ -142,7 +143,7 @@ func runStateTests(tests map[string]VmTest, skipTests []string) error { func runStateTest(test VmTest) error { db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, access.NewDbChainAccess(db)) for addr, account := range test.Pre { obj := StateObjectFromAccount(db, addr, account) statedb.SetStateObject(obj) diff --git a/tests/util.go b/tests/util.go index 571161683f..8a2f6f9630 100644 --- a/tests/util.go +++ b/tests/util.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -89,7 +90,7 @@ func (self Log) Topics() [][]byte { } func StateObjectFromAccount(db ethdb.Database, addr string, account Account) *state.StateObject { - obj := state.NewStateObject(common.HexToAddress(addr), db) + obj := state.NewStateObject(access.NoOdr, common.HexToAddress(addr), access.NewDbChainAccess(db)) obj.SetBalance(common.Big(account.Balance)) if common.IsHex(account.Code) { diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go index ddd14b1a3e..f6c6c4996d 100644 --- a/tests/vm_test_util.go +++ b/tests/vm_test_util.go @@ -25,6 +25,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" @@ -108,7 +109,7 @@ func BenchVmTest(p string, conf bconf, b *testing.B) error { func benchVmTest(test VmTest, env map[string]string, b *testing.B) { b.StopTimer() db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, access.NewDbChainAccess(db)) for addr, account := range test.Pre { obj := StateObjectFromAccount(db, addr, account) statedb.SetStateObject(obj) @@ -159,7 +160,7 @@ func runVmTests(tests map[string]VmTest, skipTests []string) error { func runVmTest(test VmTest) error { db, _ := ethdb.NewMemDatabase() - statedb, _ := state.New(common.Hash{}, db) + statedb, _ := state.New(common.Hash{}, access.NewDbChainAccess(db)) for addr, account := range test.Pre { obj := StateObjectFromAccount(db, addr, account) statedb.SetStateObject(obj) diff --git a/trie/arc.go b/trie/arc.go index 9da012e168..229420a538 100644 --- a/trie/arc.go +++ b/trie/arc.go @@ -62,6 +62,17 @@ func newARC(c int) *arc { } } +func (a *arc) Clear() { + a.mutex.Lock() + defer a.mutex.Unlock() + a.p = 0 + a.t1 = list.New() + a.b1 = list.New() + a.t2 = list.New() + a.b2 = list.New() + a.cache = make(map[string]*entry, a.c) +} + // Put inserts a new key-value pair into the cache. // This optimizes future access to this entry (side effect). func (a *arc) Put(key hashNode, value node) bool { diff --git a/trie/encoding.go b/trie/encoding.go index 3c172b8438..98ad8ee8b3 100644 --- a/trie/encoding.go +++ b/trie/encoding.go @@ -58,6 +58,7 @@ func compactDecode(str []byte) []byte { return base } +// compactHexDecode decodes a series of nibbles from a byte array func compactHexDecode(str []byte) []byte { l := len(str)*2 + 1 var nibbles = make([]byte, l) @@ -69,6 +70,24 @@ func compactHexDecode(str []byte) []byte { return nibbles } +// compactHexEncode encodes a series of nibbles into a byte array +func compactHexEncode(nibbles []byte) []byte { + nl := len(nibbles) + if nibbles[nl-1] == 16 { + nl-- + } + l := (nl + 1) / 2 + var str = make([]byte, l) + for i, _ := range str { + b := nibbles[i*2] * 16 + if nl > i*2 { + b += nibbles[i*2+1] + } + str[i] = b + } + return str +} + func decodeCompact(key []byte) []byte { l := len(key) / 2 var res = make([]byte, l) diff --git a/trie/encoding_test.go b/trie/encoding_test.go index 061d48d58b..2f125ef2f8 100644 --- a/trie/encoding_test.go +++ b/trie/encoding_test.go @@ -57,6 +57,12 @@ func (s *TrieEncodingSuite) TestCompactHexDecode(c *checker.C) { c.Assert(res, checker.DeepEquals, exp) } +func (s *TrieEncodingSuite) TestCompactHexEncode(c *checker.C) { + exp := []byte("verb") + res := compactHexEncode([]byte{7, 6, 6, 5, 7, 2, 6, 2, 16}) + c.Assert(res, checker.DeepEquals, exp) +} + func (s *TrieEncodingSuite) TestCompactDecode(c *checker.C) { // odd compact decode exp := []byte{1, 2, 3, 4, 5} diff --git a/trie/iterator.go b/trie/iterator.go index 38555fe08d..3f1d0b0d28 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -100,7 +100,7 @@ func (self *Iterator) next(node interface{}, key []byte, isIterStart bool) []byt } case hashNode: - return self.next(self.trie.resolveHash(node), key, isIterStart) + return self.next(self.trie.resolveHash(node, nil, nil), key, isIterStart) } return nil } @@ -127,7 +127,7 @@ func (self *Iterator) key(node interface{}) []byte { } } case hashNode: - return self.key(self.trie.resolveHash(node)) + return self.key(self.trie.resolveHash(node, nil, nil)) } return nil diff --git a/trie/proof.go b/trie/proof.go index a705c49db0..830bc486e1 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -10,6 +10,8 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +type MerkleProof []rlp.RawValue + // Prove constructs a merkle proof for key. The result contains all // encoded nodes on the path to the value at key. The value itself is // also included in the last node and can be retrieved by verifying @@ -17,29 +19,28 @@ import ( // // The returned proof is nil if the trie does not contain a value for key. // For existing keys, the proof will have at least one element. -func (t *Trie) Prove(key []byte) []rlp.RawValue { +func (t *Trie) Prove(key []byte) MerkleProof { // Collect all nodes on the path to key. key = compactHexDecode(key) nodes := []node{} tn := t.root - for len(key) > 0 { + for len(key) > 0 && tn != nil { switch n := tn.(type) { case shortNode: if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) { // The trie doesn't contain the key. - return nil + tn = nil + } else { + tn = n.Val } - tn = n.Val key = key[len(n.Key):] nodes = append(nodes, n) case fullNode: tn = n[key[0]] key = key[1:] nodes = append(nodes, n) - case nil: - return nil case hashNode: - tn = t.resolveHash(n) + tn = t.resolveHash(n, nil, nil) default: panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) } @@ -67,7 +68,7 @@ func (t *Trie) Prove(key []byte) []rlp.RawValue { // value for key in a trie with the given root hash. VerifyProof // returns an error if the proof contains invalid trie nodes or the // wrong value. -func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value []byte, err error) { +func VerifyProof(rootHash common.Hash, key []byte, proof MerkleProof) (value []byte, err error) { key = compactHexDecode(key) sha := sha3.NewKeccak256() wantHash := rootHash.Bytes() @@ -84,7 +85,12 @@ func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value keyrest, cld := get(n, key) switch cld := cld.(type) { case nil: - return nil, fmt.Errorf("key mismatch at proof node %d", i) + if i != len(proof)-1 { + return nil, fmt.Errorf("key mismatch at proof node %d", i) + } else { + // The trie doesn't contain the key. + return nil, nil + } case hashNode: key = keyrest wantHash = cld @@ -98,6 +104,20 @@ func VerifyProof(rootHash common.Hash, key []byte, proof []rlp.RawValue) (value return nil, errors.New("unexpected end of proof") } +// StoreProof stores the new trie nodes obtained from a merkle proof in the database +func StoreProof(db Database, proof MerkleProof) { + sha := sha3.NewKeccak256() + for _, buf := range proof { + sha.Reset() + sha.Write(buf) + hash := sha.Sum(nil) + val, _ := db.Get(hash) + if val == nil { + db.Put(hash, buf) + } + } +} + func get(tn node, key []byte) ([]byte, node) { for len(key) > 0 { switch n := tn.(type) { diff --git a/trie/proof_test.go b/trie/proof_test.go index 6b5bef05c4..6738a50bcf 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" ) @@ -33,6 +34,33 @@ func TestProof(t *testing.T) { } } +func TestStoreProof(t *testing.T) { + trie, vals := randomTrie(500) + root := trie.Hash() + mdb, _ := ethdb.NewMemDatabase() + for _, kv := range vals { + proof := trie.Prove(kv.k) + if proof == nil { + t.Fatalf("missing key %x while constructing proof", kv.k) + } + val, err := VerifyProof(root, kv.k, proof) + if err != nil { + t.Fatalf("VerifyProof error for key %x: %v\nraw proof: %x", kv.k, err, proof) + } + if !bytes.Equal(val, kv.v) { + t.Fatalf("VerifyProof returned wrong value for key %x: got %x, want %x", kv.k, val, kv.v) + } + StoreProof(mdb, proof) + } + ClearGlobalCache() + mtrie, _ := New(root, mdb) + for _, kv := range vals { + if !bytes.Equal(mtrie.Get(kv.k), kv.v) { + t.Fatalf("Can't retrieve stored proof") + } + } +} + func TestOneElementProof(t *testing.T) { trie := new(Trie) updateString(trie, "k", "v") diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 47d1934d05..3d944ed772 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -20,7 +20,9 @@ import ( "hash" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/crypto/sha3" + "golang.org/x/net/context" ) var secureKeyPrefix = []byte("secure-key-") @@ -50,16 +52,32 @@ type SecureTrie struct { // and returns ErrMissingRoot if the root node cannpt be found. // Accessing the trie loads nodes from db on demand. func NewSecure(root common.Hash, db Database) (*SecureTrie, error) { + return NewSecureOdr(access.NoOdr, root, db, nil) +} + +// NewSecureOdr creates a trie with optional ODR. If ODR is enabled, an existing +// root node in db is not required. +func NewSecureOdr(ctx context.Context, root common.Hash, db Database, access OdrAccess) (*SecureTrie, error) { if db == nil { panic("NewSecure called with nil database") } - trie, err := New(root, db) + trie, err := NewOdr(ctx, root, db, access) if err != nil { return nil, err } return &SecureTrie{Trie: trie}, nil } +// CopySecureWithOdr creates a copy of a trie with additional ODR capability +func (t *SecureTrie) CopySecureWithOdr(ctx context.Context, access OdrAccess) *SecureTrie { + return &SecureTrie{ + Trie: t.Trie.CopyWithOdr(ctx, access), + hash: t.hash, + secKeyBuf: t.secKeyBuf, + hashKeyBuf: t.hashKeyBuf, + } +} + // Get returns the value for key stored in the trie. // The value bytes must not be modified by the caller. func (t *SecureTrie) Get(key []byte) []byte { diff --git a/trie/trie.go b/trie/trie.go index a3a383fb58..fabb7468b3 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -24,11 +24,13 @@ import ( "hash" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/access" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/net/context" ) const defaultCacheCapacity = 800 @@ -44,8 +46,18 @@ var ( emptyState = crypto.Sha3Hash(nil) ) +func ClearGlobalCache() { + globalCache.Clear() +} + var ErrMissingRoot = errors.New("missing root node") +// OdrAccess is an interface to on-demand network access layer +type OdrAccess interface { + OdrEnabled() bool // is network access actually enabled + RetrieveKey(ctx context.Context, key []byte) bool // retrieve key from network and store it in local db +} + // Database must be implemented by backing stores for the trie. type Database interface { DatabaseWriter @@ -67,8 +79,10 @@ type DatabaseWriter interface { // // Trie is not safe for concurrent use. type Trie struct { - root node - db Database + root node + db Database + access OdrAccess + ctx context.Context *hasher } @@ -79,19 +93,32 @@ type Trie struct { // New will panics if db is nil or root does not exist in the // database. Accessing the trie loads nodes from db on demand. func New(root common.Hash, db Database) (*Trie, error) { - trie := &Trie{db: db} + return NewOdr(access.NoOdr, root, db, nil) +} + +// NewOdr creates a trie with optional ODR. If ODR is enabled, an existing +// root node in db is not required. +func NewOdr(ctx context.Context, root common.Hash, db Database, access OdrAccess) (*Trie, error) { + trie := &Trie{db: db, access: access, ctx: ctx} if (root != common.Hash{}) && root != emptyRoot { if db == nil { panic("trie.New: cannot use existing root without a database") } - if v, _ := trie.db.Get(root[:]); len(v) == 0 { - return nil, ErrMissingRoot + if access == nil || !access.OdrEnabled() { + if v, _ := trie.db.Get(root[:]); len(v) == 0 { + return nil, ErrMissingRoot + } } trie.root = hashNode(root.Bytes()) } return trie, nil } +// CopyWithOdr creates a copy of a trie with additional ODR capability +func (t *Trie) CopyWithOdr(ctx context.Context, access OdrAccess) *Trie { + return &Trie{db: t.db, root: t.root, access: access, ctx: ctx} +} + // Iterator returns an iterator over all mappings in the trie. func (t *Trie) Iterator() *Iterator { return NewIterator(t) @@ -101,22 +128,23 @@ func (t *Trie) Iterator() *Iterator { // The value bytes must not be modified by the caller. func (t *Trie) Get(key []byte) []byte { key = compactHexDecode(key) + pos := 0 tn := t.root - for len(key) > 0 { + for pos < len(key) { switch n := tn.(type) { case shortNode: - if len(key) < len(n.Key) || !bytes.Equal(n.Key, key[:len(n.Key)]) { + if len(key)-pos < len(n.Key) || !bytes.Equal(n.Key, key[pos:pos+len(n.Key)]) { return nil } tn = n.Val - key = key[len(n.Key):] + pos += len(n.Key) case fullNode: - tn = n[key[0]] - key = key[1:] + tn = n[key[pos]] + pos++ case nil: return nil case hashNode: - tn = t.resolveHash(n) + tn = t.resolveHash(n, key[:pos], key[pos:]) default: panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) } @@ -133,13 +161,13 @@ func (t *Trie) Get(key []byte) []byte { func (t *Trie) Update(key, value []byte) { k := compactHexDecode(key) if len(value) != 0 { - t.root = t.insert(t.root, k, valueNode(value)) + t.root = t.insert(t.root, nil, k, valueNode(value)) } else { - t.root = t.delete(t.root, k) + t.root = t.delete(t.root, nil, k) } } -func (t *Trie) insert(n node, key []byte, value node) node { +func (t *Trie) insert(n node, prefix, key []byte, value node) node { if len(key) == 0 { return value } @@ -149,12 +177,12 @@ func (t *Trie) insert(n node, key []byte, value node) node { // If the whole key matches, keep this short node as is // and only update the value. if matchlen == len(n.Key) { - return shortNode{n.Key, t.insert(n.Val, key[matchlen:], value)} + return shortNode{n.Key, t.insert(n.Val, append(prefix, key[:matchlen]...), key[matchlen:], value)} } // Otherwise branch out at the index where they differ. var branch fullNode - branch[n.Key[matchlen]] = t.insert(nil, n.Key[matchlen+1:], n.Val) - branch[key[matchlen]] = t.insert(nil, key[matchlen+1:], value) + branch[n.Key[matchlen]] = t.insert(nil, append(prefix, n.Key[:matchlen+1]...), n.Key[matchlen+1:], n.Val) + branch[key[matchlen]] = t.insert(nil, append(prefix, key[:matchlen+1]...), key[matchlen+1:], value) // Replace this shortNode with the branch if it occurs at index 0. if matchlen == 0 { return branch @@ -163,7 +191,7 @@ func (t *Trie) insert(n node, key []byte, value node) node { return shortNode{key[:matchlen], branch} case fullNode: - n[key[0]] = t.insert(n[key[0]], key[1:], value) + n[key[0]] = t.insert(n[key[0]], append(prefix, key[0]), key[1:], value) return n case nil: @@ -176,7 +204,7 @@ func (t *Trie) insert(n node, key []byte, value node) node { // // TODO: track whether insertion changed the value and keep // n as a hash node if it didn't. - return t.insert(t.resolveHash(n), key, value) + return t.insert(t.resolveHash(n, prefix, key), prefix, key, value) default: panic(fmt.Sprintf("%T: invalid node: %v", n, n)) @@ -186,13 +214,13 @@ func (t *Trie) insert(n node, key []byte, value node) node { // Delete removes any existing value for key from the trie. func (t *Trie) Delete(key []byte) { k := compactHexDecode(key) - t.root = t.delete(t.root, k) + t.root = t.delete(t.root, nil, k) } // delete returns the new root of the trie with key deleted. // It reduces the trie to minimal form by simplifying // nodes on the way up after deleting recursively. -func (t *Trie) delete(n node, key []byte) node { +func (t *Trie) delete(n node, prefix, key []byte) node { switch n := n.(type) { case shortNode: matchlen := prefixLen(key, n.Key) @@ -206,7 +234,7 @@ func (t *Trie) delete(n node, key []byte) node { // from the subtrie. Child can never be nil here since the // subtrie must contain at least two other values with keys // longer than n.Key. - child := t.delete(n.Val, key[len(n.Key):]) + child := t.delete(n.Val, append(prefix, key[:len(n.Key)]...), key[len(n.Key):]) switch child := child.(type) { case shortNode: // Deleting from the subtrie reduced it to another @@ -221,7 +249,7 @@ func (t *Trie) delete(n node, key []byte) node { } case fullNode: - n[key[0]] = t.delete(n[key[0]], key[1:]) + n[key[0]] = t.delete(n[key[0]], append(prefix, key[0]), key[1:]) // Check how many non-nil entries are left after deleting and // reduce the full node to a short node if only one entry is // left. Since n must've contained at least two children @@ -250,7 +278,7 @@ func (t *Trie) delete(n node, key []byte) node { // shortNode{..., shortNode{...}}. Since the entry // might not be loaded yet, resolve it just for this // check. - cnode := t.resolve(n[pos]) + cnode := t.resolve(n[pos], prefix, []byte{byte(pos)}) if cnode, ok := cnode.(shortNode); ok { k := append([]byte{byte(pos)}, cnode.Key...) return shortNode{k, cnode.Val} @@ -273,7 +301,7 @@ func (t *Trie) delete(n node, key []byte) node { // // TODO: track whether deletion actually hit a key and keep // n as a hash node if it didn't. - return t.delete(t.resolveHash(n), key) + return t.delete(t.resolveHash(n, prefix, key), prefix, key) default: panic(fmt.Sprintf("%T: invalid node: %v (%v)", n, n, key)) @@ -287,19 +315,28 @@ func concat(s1 []byte, s2 ...byte) []byte { return r } -func (t *Trie) resolve(n node) node { +func (t *Trie) resolve(n node, prefix, suffix []byte) node { if n, ok := n.(hashNode); ok { - return t.resolveHash(n) + return t.resolveHash(n, prefix, suffix) } return n } -func (t *Trie) resolveHash(n hashNode) node { +func (t *Trie) resolveHash(n hashNode, prefix, suffix []byte) node { if v, ok := globalCache.Get(n); ok { return v } enc, err := t.db.Get(n) if err != nil || enc == nil { + if prefix != nil || suffix != nil { + key := compactHexEncode(append(prefix, suffix...)) + if t.access != nil && t.access.RetrieveKey(t.ctx, key) { + enc, err = t.db.Get(n) + } + } + } + if err != nil || enc == nil { + // TODO: This needs to be improved to properly distinguish errors. // Disk I/O errors shouldn't produce nil (and cause a // consensus failure or weird crash), but it is unclear how @@ -348,6 +385,11 @@ func (t *Trie) Commit() (root common.Hash, err error) { // the changes made to db are written back to the trie's attached // database before using the trie. func (t *Trie) CommitTo(db DatabaseWriter) (root common.Hash, err error) { + if access.Terminated(t.ctx) { + // we should never commit potentially corrupt trie nodes to the db + return common.Hash{}, t.ctx.Err() + } + n, err := t.hashRoot(db) if err != nil { return (common.Hash{}), err diff --git a/xeth/state.go b/xeth/state.go index 981fe63b7e..e087ba8ecb 100644 --- a/xeth/state.go +++ b/xeth/state.go @@ -45,7 +45,7 @@ func (self *State) SafeGet(addr string) *Object { func (self *State) safeGet(addr string) *state.StateObject { object := self.state.GetStateObject(common.HexToAddress(addr)) if object == nil { - object = state.NewStateObject(common.HexToAddress(addr), self.xeth.backend.ChainDb()) + object = state.NewStateObject(self.state.Ctx(), common.HexToAddress(addr), self.xeth.backend.ChainAccess()) } return object diff --git a/xeth/xeth.go b/xeth/xeth.go index 35e6dd52d8..6b1ac15a76 100644 --- a/xeth/xeth.go +++ b/xeth/xeth.go @@ -41,6 +41,7 @@ import ( "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/net/context" ) var ( @@ -84,6 +85,7 @@ type XEth struct { state *State whisper *Whisper filterManager *filters.FilterSystem + ctx context.Context } func NewTest(eth *eth.Ethereum, frontend Frontend) *XEth { @@ -207,15 +209,15 @@ func (self *XEth) AtStateNum(num int64) *XEth { var err error switch num { case -2: - st = self.backend.Miner().PendingState().Copy() + st = self.backend.Miner().PendingState().CopyOdr(self.ctx) default: if block := self.getBlockByHeight(num); block != nil { - st, err = state.New(block.Root(), self.backend.ChainDb()) + st, err = state.NewOdr(self.ctx, block.Root(), self.backend.ChainAccess()) if err != nil { return nil } } else { - st, err = state.New(self.backend.BlockChain().GetBlockByNumber(0).Root(), self.backend.ChainDb()) + st, err = state.NewOdr(self.ctx, self.backend.BlockChain().GetBlockByNumberOdr(self.ctx, 0).Root(), self.backend.ChainAccess()) if err != nil { return nil } @@ -238,6 +240,19 @@ func (self *XEth) WithState(statedb *state.StateDB) *XEth { func (self *XEth) State() *State { return self.state } +// WithCtx creates a copy of the XEth object with a new ODR context +func (self *XEth) WithCtx(ctx context.Context) *XEth { + xeth := &XEth{ + backend: self.backend, + frontend: self.frontend, + gpo: self.gpo, + ctx: ctx, + } + + xeth.state = NewState(xeth, self.state.State().CopyOdr(ctx)) + return xeth +} + // subscribes to new head block events and // waits until blockchain height is greater n at any time // given the current head, waits for the next chain event @@ -270,7 +285,7 @@ func (self *XEth) UpdateState() (wait chan *big.Int) { wait <- n n = nil } - statedb, err := state.New(event.Block.Root(), self.backend.ChainDb()) + statedb, err := state.New(event.Block.Root(), self.backend.ChainAccess()) if err != nil { glog.V(logger.Error).Infoln("Could not create new state: %v", err) return @@ -305,19 +320,19 @@ func (self *XEth) getBlockByHeight(height int64) *types.Block { num = uint64(height) } - return self.backend.BlockChain().GetBlockByNumber(num) + return self.backend.BlockChain().GetBlockByNumberOdr(self.ctx, num) } func (self *XEth) BlockByHash(strHash string) *Block { hash := common.HexToHash(strHash) - block := self.backend.BlockChain().GetBlock(hash) + block := self.backend.BlockChain().GetBlockOdr(self.ctx, hash) return NewBlock(block) } func (self *XEth) EthBlockByHash(strHash string) *types.Block { hash := common.HexToHash(strHash) - block := self.backend.BlockChain().GetBlock(hash) + block := self.backend.BlockChain().GetBlockOdr(self.ctx, hash) return block } @@ -379,11 +394,11 @@ func (self *XEth) CurrentBlock() *types.Block { } func (self *XEth) GetBlockReceipts(bhash common.Hash) types.Receipts { - return self.backend.BlockProcessor().GetBlockReceipts(bhash) + return self.backend.BlockProcessor().GetBlockReceipts(bhash) //ODR } -func (self *XEth) GetTxReceipt(txhash common.Hash) *types.Receipt { - return core.GetReceipt(self.backend.ChainDb(), txhash) +func (self *XEth) GetTxReceipt(txhash common.Hash) *types.Receipt { //ODR + return core.GetReceipt(self.backend.ChainAccess(), txhash) //TODO } func (self *XEth) GasLimit() *big.Int { @@ -547,7 +562,7 @@ func (self *XEth) NewLogFilter(earliest, latest int64, skip, max int, address [] self.logMu.Lock() defer self.logMu.Unlock() - filter := filters.New(self.backend.ChainDb()) + filter := filters.New(self.backend.ChainAccess()) id := self.filterManager.Add(filter) self.logQueue[id] = &logQueue{timeout: time.Now()} @@ -571,7 +586,7 @@ func (self *XEth) NewTransactionFilter() int { self.transactionMu.Lock() defer self.transactionMu.Unlock() - filter := filters.New(self.backend.ChainDb()) + filter := filters.New(self.backend.ChainAccess()) id := self.filterManager.Add(filter) self.transactionQueue[id] = &hashQueue{timeout: time.Now()} @@ -590,7 +605,7 @@ func (self *XEth) NewBlockFilter() int { self.blockMu.Lock() defer self.blockMu.Unlock() - filter := filters.New(self.backend.ChainDb()) + filter := filters.New(self.backend.ChainAccess()) id := self.filterManager.Add(filter) self.blockQueue[id] = &hashQueue{timeout: time.Now()} @@ -657,7 +672,7 @@ func (self *XEth) Logs(id int) vm.Logs { } func (self *XEth) AllLogs(earliest, latest int64, skip, max int, address []string, topics [][]string) vm.Logs { - filter := filters.New(self.backend.ChainDb()) + filter := filters.New(self.backend.ChainAccess()) filter.SetBeginBlock(earliest) filter.SetEndBlock(latest) filter.SetAddresses(cAddress(address)) @@ -829,7 +844,7 @@ func (self *XEth) PushTx(encodedTx string) (string, error) { } func (self *XEth) Call(fromStr, toStr, valueStr, gasStr, gasPriceStr, dataStr string) (string, string, error) { - statedb := self.State().State().Copy() + statedb := self.State().State().CopyWithCtx() var from *state.StateObject if len(fromStr) == 0 { accounts, err := self.backend.AccountManager().Accounts()