From aca588a8e434cd0409c44d55b6009fb74e913537 Mon Sep 17 00:00:00 2001 From: Jeremy Schlatter Date: Mon, 7 Jan 2019 00:35:44 -0800 Subject: [PATCH 01/14] accounts/keystore: small code simplification (#18394) --- accounts/keystore/wallet.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/accounts/keystore/wallet.go b/accounts/keystore/wallet.go index 60044b1f9a..2f774cc941 100644 --- a/accounts/keystore/wallet.go +++ b/accounts/keystore/wallet.go @@ -84,10 +84,7 @@ func (w *keystoreWallet) SelfDerive(base accounts.DerivationPath, chain ethereum // able to sign via our shared keystore backend). func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) { // Make sure the requested account is contained within - if account.Address != w.account.Address { - return nil, accounts.ErrUnknownAccount - } - if account.URL != (accounts.URL{}) && account.URL != w.account.URL { + if !w.Contains(account) { return nil, accounts.ErrUnknownAccount } // Account seems valid, request the keystore to sign @@ -100,10 +97,7 @@ func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte // be able to sign via our shared keystore backend). func (w *keystoreWallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { // Make sure the requested account is contained within - if account.Address != w.account.Address { - return nil, accounts.ErrUnknownAccount - } - if account.URL != (accounts.URL{}) && account.URL != w.account.URL { + if !w.Contains(account) { return nil, accounts.ErrUnknownAccount } // Account seems valid, request the keystore to sign @@ -114,10 +108,7 @@ func (w *keystoreWallet) SignTx(account accounts.Account, tx *types.Transaction, // given hash with the given account using passphrase as extra authentication. func (w *keystoreWallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) { // Make sure the requested account is contained within - if account.Address != w.account.Address { - return nil, accounts.ErrUnknownAccount - } - if account.URL != (accounts.URL{}) && account.URL != w.account.URL { + if !w.Contains(account) { return nil, accounts.ErrUnknownAccount } // Account seems valid, request the keystore to sign @@ -128,10 +119,7 @@ func (w *keystoreWallet) SignHashWithPassphrase(account accounts.Account, passph // transaction with the given account using passphrase as extra authentication. func (w *keystoreWallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { // Make sure the requested account is contained within - if account.Address != w.account.Address { - return nil, accounts.ErrUnknownAccount - } - if account.URL != (accounts.URL{}) && account.URL != w.account.URL { + if !w.Contains(account) { return nil, accounts.ErrUnknownAccount } // Account seems valid, request the keystore to sign From e05d46807525f76dab83f84e9dfc9e4537398641 Mon Sep 17 00:00:00 2001 From: Yondon Fu Date: Mon, 7 Jan 2019 03:47:11 -0500 Subject: [PATCH 02/14] internal/ethapi: ask transaction pool for pending nonce (#15794) --- internal/ethapi/api.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 656555b3ba..73b629bd9c 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1074,6 +1074,15 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByBlockHashAndIndex(ctx cont // GetTransactionCount returns the number of transactions the given address has sent for the given block number func (s *PublicTransactionPoolAPI) GetTransactionCount(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (*hexutil.Uint64, error) { + // Ask transaction pool for the nonce which includes pending transactions + if blockNr == rpc.PendingBlockNumber { + nonce, err := s.b.GetPoolNonce(ctx, address) + if err != nil { + return nil, err + } + return (*hexutil.Uint64)(&nonce), nil + } + // Resolve block number and use its state to ask for the nonce state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr) if state == nil || err != nil { return nil, err From 428eabe28d1077356b16f25828d78d8693a766bb Mon Sep 17 00:00:00 2001 From: Sean Date: Mon, 7 Jan 2019 19:56:50 +1100 Subject: [PATCH 03/14] cmd/geth: support dumpconfig optionally saving to file (#18327) * Changed dumpConfig function to optionally save to file * Added O_TRUNC flag to file open and cleaned up code --- cmd/geth/config.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 59f759f0ea..f1e2811966 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -20,7 +20,6 @@ import ( "bufio" "errors" "fmt" - "io" "math/big" "os" "reflect" @@ -198,7 +197,17 @@ func dumpConfig(ctx *cli.Context) error { if err != nil { return err } - io.WriteString(os.Stdout, comment) - os.Stdout.Write(out) + + dump := os.Stdout + if ctx.NArg() > 0 { + dump, err = os.OpenFile(ctx.Args().Get(0), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer dump.Close() + } + dump.WriteString(comment) + dump.Write(out) + return nil } From 356c49fa7ec88632f839226c9b0f1cf172ec6f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Mon, 7 Jan 2019 13:20:11 +0100 Subject: [PATCH 04/14] swarm: Shed Index and Uint64Field additions (#18398) --- swarm/shed/db.go | 2 +- swarm/shed/example_store_test.go | 40 +-- swarm/shed/field_uint64.go | 38 +++ swarm/shed/field_uint64_test.go | 106 +++++++ swarm/shed/index.go | 156 ++++++---- swarm/shed/index_test.go | 475 +++++++++++++++++++++++++++---- 6 files changed, 679 insertions(+), 138 deletions(-) diff --git a/swarm/shed/db.go b/swarm/shed/db.go index 7377e12d2d..d4e5d1b231 100644 --- a/swarm/shed/db.go +++ b/swarm/shed/db.go @@ -18,7 +18,7 @@ // more complex operations on storage data organized in fields and indexes. // // Only type which holds logical information about swarm storage chunks data -// and metadata is IndexItem. This part is not generalized mostly for +// and metadata is Item. This part is not generalized mostly for // performance reasons. package shed diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go index 908a1e446d..9a83855e70 100644 --- a/swarm/shed/example_store_test.go +++ b/swarm/shed/example_store_test.go @@ -71,20 +71,20 @@ func New(path string) (s *Store, err error) { } // Index storing actual chunk address, data and store timestamp. s.retrievalIndex, err = db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{ - EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + EncodeKey: func(fields shed.Item) (key []byte, err error) { return fields.Address, nil }, - DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + DecodeKey: func(key []byte) (e shed.Item, err error) { e.Address = key return e, nil }, - EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + EncodeValue: func(fields shed.Item) (value []byte, err error) { b := make([]byte, 8) binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) value = append(b, fields.Data...) return value, nil }, - DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) e.Data = value[8:] return e, nil @@ -96,19 +96,19 @@ func New(path string) (s *Store, err error) { // Index storing access timestamp for a particular address. // It is needed in order to update gc index keys for iteration order. s.accessIndex, err = db.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{ - EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + EncodeKey: func(fields shed.Item) (key []byte, err error) { return fields.Address, nil }, - DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + DecodeKey: func(key []byte) (e shed.Item, err error) { e.Address = key return e, nil }, - EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + EncodeValue: func(fields shed.Item) (value []byte, err error) { b := make([]byte, 8) binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) return b, nil }, - DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) return e, nil }, @@ -118,23 +118,23 @@ func New(path string) (s *Store, err error) { } // Index with keys ordered by access timestamp for garbage collection prioritization. s.gcIndex, err = db.NewIndex("AccessTimestamp|StoredTimestamp|Address->nil", shed.IndexFuncs{ - EncodeKey: func(fields shed.IndexItem) (key []byte, err error) { + EncodeKey: func(fields shed.Item) (key []byte, err error) { b := make([]byte, 16, 16+len(fields.Address)) binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) key = append(b, fields.Address...) return key, nil }, - DecodeKey: func(key []byte) (e shed.IndexItem, err error) { + DecodeKey: func(key []byte) (e shed.Item, err error) { e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) e.Address = key[16:] return e, nil }, - EncodeValue: func(fields shed.IndexItem) (value []byte, err error) { + EncodeValue: func(fields shed.Item) (value []byte, err error) { return nil, nil }, - DecodeValue: func(value []byte) (e shed.IndexItem, err error) { + DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { return e, nil }, }) @@ -146,7 +146,7 @@ func New(path string) (s *Store, err error) { // Put stores the chunk and sets it store timestamp. func (s *Store) Put(_ context.Context, ch storage.Chunk) (err error) { - return s.retrievalIndex.Put(shed.IndexItem{ + return s.retrievalIndex.Put(shed.Item{ Address: ch.Address(), Data: ch.Data(), StoreTimestamp: time.Now().UTC().UnixNano(), @@ -161,7 +161,7 @@ func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, e batch := new(leveldb.Batch) // Get the chunk data and storage timestamp. - item, err := s.retrievalIndex.Get(shed.IndexItem{ + item, err := s.retrievalIndex.Get(shed.Item{ Address: addr, }) if err != nil { @@ -172,13 +172,13 @@ func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, e } // Get the chunk access timestamp. - accessItem, err := s.accessIndex.Get(shed.IndexItem{ + accessItem, err := s.accessIndex.Get(shed.Item{ Address: addr, }) switch err { case nil: // Remove gc index entry if access timestamp is found. - err = s.gcIndex.DeleteInBatch(batch, shed.IndexItem{ + err = s.gcIndex.DeleteInBatch(batch, shed.Item{ Address: item.Address, StoreTimestamp: accessItem.AccessTimestamp, AccessTimestamp: item.StoreTimestamp, @@ -197,7 +197,7 @@ func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, e accessTimestamp := time.Now().UTC().UnixNano() // Put new access timestamp in access index. - err = s.accessIndex.PutInBatch(batch, shed.IndexItem{ + err = s.accessIndex.PutInBatch(batch, shed.Item{ Address: addr, AccessTimestamp: accessTimestamp, }) @@ -206,7 +206,7 @@ func (s *Store) Get(_ context.Context, addr storage.Address) (c storage.Chunk, e } // Put new access timestamp in gc index. - err = s.gcIndex.PutInBatch(batch, shed.IndexItem{ + err = s.gcIndex.PutInBatch(batch, shed.Item{ Address: item.Address, AccessTimestamp: accessTimestamp, StoreTimestamp: item.StoreTimestamp, @@ -244,7 +244,7 @@ func (s *Store) CollectGarbage() (err error) { // New batch for a new cg round. trash := new(leveldb.Batch) // Iterate through all index items and break when needed. - err = s.gcIndex.IterateAll(func(item shed.IndexItem) (stop bool, err error) { + err = s.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) { // Remove the chunk. err = s.retrievalIndex.DeleteInBatch(trash, item) if err != nil { @@ -265,7 +265,7 @@ func (s *Store) CollectGarbage() (err error) { return true, nil } return false, nil - }) + }, nil) if err != nil { return err } diff --git a/swarm/shed/field_uint64.go b/swarm/shed/field_uint64.go index 80e0069ae4..0417583ac3 100644 --- a/swarm/shed/field_uint64.go +++ b/swarm/shed/field_uint64.go @@ -99,6 +99,44 @@ func (f Uint64Field) IncInBatch(batch *leveldb.Batch) (val uint64, err error) { return val, nil } +// Dec decrements a uint64 value in the database. +// This operation is not goroutine save. +// The field is protected from overflow to a negative value. +func (f Uint64Field) Dec() (val uint64, err error) { + val, err = f.Get() + if err != nil { + if err == leveldb.ErrNotFound { + val = 0 + } else { + return 0, err + } + } + if val != 0 { + val-- + } + return val, f.Put(val) +} + +// DecInBatch decrements a uint64 value in the batch +// by retreiving a value from the database, not the same batch. +// This operation is not goroutine save. +// The field is protected from overflow to a negative value. +func (f Uint64Field) DecInBatch(batch *leveldb.Batch) (val uint64, err error) { + val, err = f.Get() + if err != nil { + if err == leveldb.ErrNotFound { + val = 0 + } else { + return 0, err + } + } + if val != 0 { + val-- + } + f.PutInBatch(batch, val) + return val, nil +} + // encode transforms uint64 to 8 byte long // slice in big endian encoding. func encodeUint64(val uint64) (b []byte) { diff --git a/swarm/shed/field_uint64_test.go b/swarm/shed/field_uint64_test.go index 69ade71ba3..9462b56dd1 100644 --- a/swarm/shed/field_uint64_test.go +++ b/swarm/shed/field_uint64_test.go @@ -192,3 +192,109 @@ func TestUint64Field_IncInBatch(t *testing.T) { t.Errorf("got uint64 %v, want %v", got, want) } } + +// TestUint64Field_Dec validates Dec operation +// of the Uint64Field. +func TestUint64Field_Dec(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + counter, err := db.NewUint64Field("counter") + if err != nil { + t.Fatal(err) + } + + // test overflow protection + var want uint64 + got, err := counter.Dec() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + want = 32 + err = counter.Put(want) + if err != nil { + t.Fatal(err) + } + + want = 31 + got, err = counter.Dec() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } +} + +// TestUint64Field_DecInBatch validates DecInBatch operation +// of the Uint64Field. +func TestUint64Field_DecInBatch(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + counter, err := db.NewUint64Field("counter") + if err != nil { + t.Fatal(err) + } + + batch := new(leveldb.Batch) + var want uint64 + got, err := counter.DecInBatch(batch) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + got, err = counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + batch2 := new(leveldb.Batch) + want = 42 + counter.PutInBatch(batch2, want) + err = db.WriteBatch(batch2) + if err != nil { + t.Fatal(err) + } + got, err = counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + + batch3 := new(leveldb.Batch) + want = 41 + got, err = counter.DecInBatch(batch3) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } + err = db.WriteBatch(batch3) + if err != nil { + t.Fatal(err) + } + got, err = counter.Get() + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("got uint64 %v, want %v", got, want) + } +} diff --git a/swarm/shed/index.go b/swarm/shed/index.go index ba803e3c23..df88b1b62d 100644 --- a/swarm/shed/index.go +++ b/swarm/shed/index.go @@ -17,22 +17,24 @@ package shed import ( + "bytes" + "github.com/syndtr/goleveldb/leveldb" ) -// IndexItem holds fields relevant to Swarm Chunk data and metadata. +// Item holds fields relevant to Swarm Chunk data and metadata. // All information required for swarm storage and operations // on that storage must be defined here. // This structure is logically connected to swarm storage, // the only part of this package that is not generalized, // mostly for performance reasons. // -// IndexItem is a type that is used for retrieving, storing and encoding +// Item is a type that is used for retrieving, storing and encoding // chunk data and metadata. It is passed as an argument to Index encoding // functions, get function and put function. // But it is also returned with additional data from get function call // and as the argument in iterator function definition. -type IndexItem struct { +type Item struct { Address []byte Data []byte AccessTimestamp int64 @@ -43,9 +45,9 @@ type IndexItem struct { } // Merge is a helper method to construct a new -// IndexItem by filling up fields with default values -// of a particular IndexItem with values from another one. -func (i IndexItem) Merge(i2 IndexItem) (new IndexItem) { +// Item by filling up fields with default values +// of a particular Item with values from another one. +func (i Item) Merge(i2 Item) (new Item) { if i.Address == nil { i.Address = i2.Address } @@ -67,26 +69,26 @@ func (i IndexItem) Merge(i2 IndexItem) (new IndexItem) { // Index represents a set of LevelDB key value pairs that have common // prefix. It holds functions for encoding and decoding keys and values // to provide transparent actions on saved data which inclide: -// - getting a particular IndexItem -// - saving a particular IndexItem +// - getting a particular Item +// - saving a particular Item // - iterating over a sorted LevelDB keys // It implements IndexIteratorInterface interface. type Index struct { db *DB prefix []byte - encodeKeyFunc func(fields IndexItem) (key []byte, err error) - decodeKeyFunc func(key []byte) (e IndexItem, err error) - encodeValueFunc func(fields IndexItem) (value []byte, err error) - decodeValueFunc func(value []byte) (e IndexItem, err error) + encodeKeyFunc func(fields Item) (key []byte, err error) + decodeKeyFunc func(key []byte) (e Item, err error) + encodeValueFunc func(fields Item) (value []byte, err error) + decodeValueFunc func(keyFields Item, value []byte) (e Item, err error) } // IndexFuncs structure defines functions for encoding and decoding // LevelDB keys and values for a specific index. type IndexFuncs struct { - EncodeKey func(fields IndexItem) (key []byte, err error) - DecodeKey func(key []byte) (e IndexItem, err error) - EncodeValue func(fields IndexItem) (value []byte, err error) - DecodeValue func(value []byte) (e IndexItem, err error) + EncodeKey func(fields Item) (key []byte, err error) + DecodeKey func(key []byte) (e Item, err error) + EncodeValue func(fields Item) (value []byte, err error) + DecodeValue func(keyFields Item, value []byte) (e Item, err error) } // NewIndex returns a new Index instance with defined name and @@ -105,7 +107,7 @@ func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { // by appending the provided index id byte. // This is needed to avoid collisions between keys of different // indexes as all index ids are unique. - encodeKeyFunc: func(e IndexItem) (key []byte, err error) { + encodeKeyFunc: func(e Item) (key []byte, err error) { key, err = funcs.EncodeKey(e) if err != nil { return nil, err @@ -115,7 +117,7 @@ func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { // This function reverses the encodeKeyFunc constructed key // to transparently work with index keys without their index ids. // It assumes that index keys are prefixed with only one byte. - decodeKeyFunc: func(key []byte) (e IndexItem, err error) { + decodeKeyFunc: func(key []byte) (e Item, err error) { return funcs.DecodeKey(key[1:]) }, encodeValueFunc: funcs.EncodeValue, @@ -123,10 +125,10 @@ func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) { }, nil } -// Get accepts key fields represented as IndexItem to retrieve a +// Get accepts key fields represented as Item to retrieve a // value from the index and return maximum available information -// from the index represented as another IndexItem. -func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) { +// from the index represented as another Item. +func (f Index) Get(keyFields Item) (out Item, err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { return out, err @@ -135,16 +137,16 @@ func (f Index) Get(keyFields IndexItem) (out IndexItem, err error) { if err != nil { return out, err } - out, err = f.decodeValueFunc(value) + out, err = f.decodeValueFunc(keyFields, value) if err != nil { return out, err } return out.Merge(keyFields), nil } -// Put accepts IndexItem to encode information from it +// Put accepts Item to encode information from it // and save it to the database. -func (f Index) Put(i IndexItem) (err error) { +func (f Index) Put(i Item) (err error) { key, err := f.encodeKeyFunc(i) if err != nil { return err @@ -159,7 +161,7 @@ func (f Index) Put(i IndexItem) (err error) { // PutInBatch is the same as Put method, but it just // saves the key/value pair to the batch instead // directly to the database. -func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) { +func (f Index) PutInBatch(batch *leveldb.Batch, i Item) (err error) { key, err := f.encodeKeyFunc(i) if err != nil { return err @@ -172,9 +174,9 @@ func (f Index) PutInBatch(batch *leveldb.Batch, i IndexItem) (err error) { return nil } -// Delete accepts IndexItem to remove a key/value pair +// Delete accepts Item to remove a key/value pair // from the database based on its fields. -func (f Index) Delete(keyFields IndexItem) (err error) { +func (f Index) Delete(keyFields Item) (err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { return err @@ -184,7 +186,7 @@ func (f Index) Delete(keyFields IndexItem) (err error) { // DeleteInBatch is the same as Delete just the operation // is performed on the batch instead on the database. -func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error) { +func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields Item) (err error) { key, err := f.encodeKeyFunc(keyFields) if err != nil { return err @@ -193,32 +195,71 @@ func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err err return nil } -// IndexIterFunc is a callback on every IndexItem that is decoded +// IndexIterFunc is a callback on every Item that is decoded // by iterating on an Index keys. // By returning a true for stop variable, iteration will // stop, and by returning the error, that error will be // propagated to the called iterator method on Index. -type IndexIterFunc func(item IndexItem) (stop bool, err error) +type IndexIterFunc func(item Item) (stop bool, err error) -// IterateAll iterates over all keys of the Index. -func (f Index) IterateAll(fn IndexIterFunc) (err error) { +// IterateOptions defines optional parameters for Iterate function. +type IterateOptions struct { + // StartFrom is the Item to start the iteration from. + StartFrom *Item + // If SkipStartFromItem is true, StartFrom item will not + // be iterated on. + SkipStartFromItem bool + // Iterate over items which keys have a common prefix. + Prefix []byte +} + +// Iterate function iterates over keys of the Index. +// If IterateOptions is nil, the iterations is over all keys. +func (f Index) Iterate(fn IndexIterFunc, options *IterateOptions) (err error) { + if options == nil { + options = new(IterateOptions) + } + // construct a prefix with Index prefix and optional common key prefix + prefix := append(f.prefix, options.Prefix...) + // start from the prefix + startKey := prefix + if options.StartFrom != nil { + // start from the provided StartFrom Item key value + startKey, err = f.encodeKeyFunc(*options.StartFrom) + if err != nil { + return err + } + } it := f.db.NewIterator() defer it.Release() - for ok := it.Seek(f.prefix); ok; ok = it.Next() { + // move the cursor to the start key + ok := it.Seek(startKey) + if !ok { + // stop iterator if seek has failed + return it.Error() + } + if options.SkipStartFromItem && bytes.Equal(startKey, it.Key()) { + // skip the start from Item if it is the first key + // and it is explicitly configured to skip it + ok = it.Next() + } + for ; ok; ok = it.Next() { key := it.Key() - if key[0] != f.prefix[0] { + if !bytes.HasPrefix(key, prefix) { break } - keyIndexItem, err := f.decodeKeyFunc(key) + // create a copy of key byte slice not to share leveldb underlaying slice array + keyItem, err := f.decodeKeyFunc(append([]byte(nil), key...)) if err != nil { return err } - valueIndexItem, err := f.decodeValueFunc(it.Value()) + // create a copy of value byte slice not to share leveldb underlaying slice array + valueItem, err := f.decodeValueFunc(keyItem, append([]byte(nil), it.Value()...)) if err != nil { return err } - stop, err := fn(keyIndexItem.Merge(valueIndexItem)) + stop, err := fn(keyItem.Merge(valueItem)) if err != nil { return err } @@ -229,12 +270,27 @@ func (f Index) IterateAll(fn IndexIterFunc) (err error) { return it.Error() } -// IterateFrom iterates over Index keys starting from the key -// encoded from the provided IndexItem. -func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) { +// Count returns the number of items in index. +func (f Index) Count() (count int, err error) { + it := f.db.NewIterator() + defer it.Release() + + for ok := it.Seek(f.prefix); ok; ok = it.Next() { + key := it.Key() + if key[0] != f.prefix[0] { + break + } + count++ + } + return count, it.Error() +} + +// CountFrom returns the number of items in index keys +// starting from the key encoded from the provided Item. +func (f Index) CountFrom(start Item) (count int, err error) { startKey, err := f.encodeKeyFunc(start) if err != nil { - return err + return 0, err } it := f.db.NewIterator() defer it.Release() @@ -244,21 +300,7 @@ func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) { if key[0] != f.prefix[0] { break } - keyIndexItem, err := f.decodeKeyFunc(key) - if err != nil { - return err - } - valueIndexItem, err := f.decodeValueFunc(it.Value()) - if err != nil { - return err - } - stop, err := fn(keyIndexItem.Merge(valueIndexItem)) - if err != nil { - return err - } - if stop { - break - } + count++ } - return it.Error() + return count, it.Error() } diff --git a/swarm/shed/index_test.go b/swarm/shed/index_test.go index ba82216dfe..97d7c91f43 100644 --- a/swarm/shed/index_test.go +++ b/swarm/shed/index_test.go @@ -29,20 +29,20 @@ import ( // Index functions for the index that is used in tests in this file. var retrievalIndexFuncs = IndexFuncs{ - EncodeKey: func(fields IndexItem) (key []byte, err error) { + EncodeKey: func(fields Item) (key []byte, err error) { return fields.Address, nil }, - DecodeKey: func(key []byte) (e IndexItem, err error) { + DecodeKey: func(key []byte) (e Item, err error) { e.Address = key return e, nil }, - EncodeValue: func(fields IndexItem) (value []byte, err error) { + EncodeValue: func(fields Item) (value []byte, err error) { b := make([]byte, 8) binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) value = append(b, fields.Data...) return value, nil }, - DecodeValue: func(value []byte) (e IndexItem, err error) { + DecodeValue: func(keyItem Item, value []byte) (e Item, err error) { e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) e.Data = value[8:] return e, nil @@ -60,7 +60,7 @@ func TestIndex(t *testing.T) { } t.Run("put", func(t *testing.T) { - want := IndexItem{ + want := Item{ Address: []byte("put-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), @@ -70,16 +70,16 @@ func TestIndex(t *testing.T) { if err != nil { t.Fatal(err) } - got, err := index.Get(IndexItem{ + got, err := index.Get(Item{ Address: want.Address, }) if err != nil { t.Fatal(err) } - checkIndexItem(t, got, want) + checkItem(t, got, want) t.Run("overwrite", func(t *testing.T) { - want := IndexItem{ + want := Item{ Address: []byte("put-hash"), Data: []byte("New DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), @@ -89,18 +89,18 @@ func TestIndex(t *testing.T) { if err != nil { t.Fatal(err) } - got, err := index.Get(IndexItem{ + got, err := index.Get(Item{ Address: want.Address, }) if err != nil { t.Fatal(err) } - checkIndexItem(t, got, want) + checkItem(t, got, want) }) }) t.Run("put in batch", func(t *testing.T) { - want := IndexItem{ + want := Item{ Address: []byte("put-in-batch-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), @@ -112,16 +112,16 @@ func TestIndex(t *testing.T) { if err != nil { t.Fatal(err) } - got, err := index.Get(IndexItem{ + got, err := index.Get(Item{ Address: want.Address, }) if err != nil { t.Fatal(err) } - checkIndexItem(t, got, want) + checkItem(t, got, want) t.Run("overwrite", func(t *testing.T) { - want := IndexItem{ + want := Item{ Address: []byte("put-in-batch-hash"), Data: []byte("New DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), @@ -133,13 +133,13 @@ func TestIndex(t *testing.T) { if err != nil { t.Fatal(err) } - got, err := index.Get(IndexItem{ + got, err := index.Get(Item{ Address: want.Address, }) if err != nil { t.Fatal(err) } - checkIndexItem(t, got, want) + checkItem(t, got, want) }) }) @@ -150,13 +150,13 @@ func TestIndex(t *testing.T) { address := []byte("put-in-batch-twice-hash") // put the first item - index.PutInBatch(batch, IndexItem{ + index.PutInBatch(batch, Item{ Address: address, Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), }) - want := IndexItem{ + want := Item{ Address: address, Data: []byte("New DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), @@ -168,17 +168,17 @@ func TestIndex(t *testing.T) { if err != nil { t.Fatal(err) } - got, err := index.Get(IndexItem{ + got, err := index.Get(Item{ Address: address, }) if err != nil { t.Fatal(err) } - checkIndexItem(t, got, want) + checkItem(t, got, want) }) t.Run("delete", func(t *testing.T) { - want := IndexItem{ + want := Item{ Address: []byte("delete-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), @@ -188,15 +188,15 @@ func TestIndex(t *testing.T) { if err != nil { t.Fatal(err) } - got, err := index.Get(IndexItem{ + got, err := index.Get(Item{ Address: want.Address, }) if err != nil { t.Fatal(err) } - checkIndexItem(t, got, want) + checkItem(t, got, want) - err = index.Delete(IndexItem{ + err = index.Delete(Item{ Address: want.Address, }) if err != nil { @@ -204,7 +204,7 @@ func TestIndex(t *testing.T) { } wantErr := leveldb.ErrNotFound - got, err = index.Get(IndexItem{ + got, err = index.Get(Item{ Address: want.Address, }) if err != wantErr { @@ -213,7 +213,7 @@ func TestIndex(t *testing.T) { }) t.Run("delete in batch", func(t *testing.T) { - want := IndexItem{ + want := Item{ Address: []byte("delete-in-batch-hash"), Data: []byte("DATA"), StoreTimestamp: time.Now().UTC().UnixNano(), @@ -223,16 +223,16 @@ func TestIndex(t *testing.T) { if err != nil { t.Fatal(err) } - got, err := index.Get(IndexItem{ + got, err := index.Get(Item{ Address: want.Address, }) if err != nil { t.Fatal(err) } - checkIndexItem(t, got, want) + checkItem(t, got, want) batch := new(leveldb.Batch) - index.DeleteInBatch(batch, IndexItem{ + index.DeleteInBatch(batch, Item{ Address: want.Address, }) err = db.WriteBatch(batch) @@ -241,7 +241,7 @@ func TestIndex(t *testing.T) { } wantErr := leveldb.ErrNotFound - got, err = index.Get(IndexItem{ + got, err = index.Get(Item{ Address: want.Address, }) if err != wantErr { @@ -250,8 +250,9 @@ func TestIndex(t *testing.T) { }) } -// TestIndex_iterate validates index iterator functions for correctness. -func TestIndex_iterate(t *testing.T) { +// TestIndex_Iterate validates index Iterate +// functions for correctness. +func TestIndex_Iterate(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -260,7 +261,7 @@ func TestIndex_iterate(t *testing.T) { t.Fatal(err) } - items := []IndexItem{ + items := []Item{ { Address: []byte("iterate-hash-01"), Data: []byte("data80"), @@ -290,7 +291,7 @@ func TestIndex_iterate(t *testing.T) { if err != nil { t.Fatal(err) } - item04 := IndexItem{ + item04 := Item{ Address: []byte("iterate-hash-04"), Data: []byte("data0"), } @@ -306,31 +307,53 @@ func TestIndex_iterate(t *testing.T) { t.Run("all", func(t *testing.T) { var i int - err := index.IterateAll(func(item IndexItem) (stop bool, err error) { + err := index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } want := items[i] - checkIndexItem(t, item, want) + checkItem(t, item, want) i++ return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("start from", func(t *testing.T) { + startIndex := 2 + i := startIndex + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], }) if err != nil { t.Fatal(err) } }) - t.Run("from", func(t *testing.T) { + t.Run("skip start from", func(t *testing.T) { startIndex := 2 - i := startIndex - err := index.IterateFrom(items[startIndex], func(item IndexItem) (stop bool, err error) { + i := startIndex + 1 + err := index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } want := items[i] - checkIndexItem(t, item, want) + checkItem(t, item, want) i++ return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], + SkipStartFromItem: true, }) if err != nil { t.Fatal(err) @@ -341,18 +364,209 @@ func TestIndex_iterate(t *testing.T) { var i int stopIndex := 3 var count int - err := index.IterateAll(func(item IndexItem) (stop bool, err error) { + err := index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } want := items[i] - checkIndexItem(t, item, want) + checkItem(t, item, want) count++ if i == stopIndex { return true, nil } i++ return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } + wantItemsCount := stopIndex + 1 + if count != wantItemsCount { + t.Errorf("got %v items, expected %v", count, wantItemsCount) + } + }) + + t.Run("no overflow", func(t *testing.T) { + secondIndex, err := db.NewIndex("second-index", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + secondItem := Item{ + Address: []byte("iterate-hash-10"), + Data: []byte("data-second"), + } + err = secondIndex.Put(secondItem) + if err != nil { + t.Fatal(err) + } + + var i int + err = index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } + + i = 0 + err = secondIndex.Iterate(func(item Item) (stop bool, err error) { + if i > 1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + checkItem(t, item, secondItem) + i++ + return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } + }) +} + +// TestIndex_Iterate_withPrefix validates index Iterate +// function for correctness. +func TestIndex_Iterate_withPrefix(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + index, err := db.NewIndex("retrieval", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + allItems := []Item{ + {Address: []byte("want-hash-00"), Data: []byte("data80")}, + {Address: []byte("skip-hash-01"), Data: []byte("data81")}, + {Address: []byte("skip-hash-02"), Data: []byte("data82")}, + {Address: []byte("skip-hash-03"), Data: []byte("data83")}, + {Address: []byte("want-hash-04"), Data: []byte("data84")}, + {Address: []byte("want-hash-05"), Data: []byte("data85")}, + {Address: []byte("want-hash-06"), Data: []byte("data86")}, + {Address: []byte("want-hash-07"), Data: []byte("data87")}, + {Address: []byte("want-hash-08"), Data: []byte("data88")}, + {Address: []byte("want-hash-09"), Data: []byte("data89")}, + {Address: []byte("skip-hash-10"), Data: []byte("data90")}, + } + batch := new(leveldb.Batch) + for _, i := range allItems { + index.PutInBatch(batch, i) + } + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + + prefix := []byte("want") + + items := make([]Item, 0) + for _, item := range allItems { + if bytes.HasPrefix(item.Address, prefix) { + items = append(items, item) + } + } + sort.SliceStable(items, func(i, j int) bool { + return bytes.Compare(items[i].Address, items[j].Address) < 0 + }) + + t.Run("with prefix", func(t *testing.T) { + var i int + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + return false, nil + }, &IterateOptions{ + Prefix: prefix, + }) + if err != nil { + t.Fatal(err) + } + if i != len(items) { + t.Errorf("got %v items, want %v", i, len(items)) + } + }) + + t.Run("with prefix and start from", func(t *testing.T) { + startIndex := 2 + var count int + i := startIndex + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + count++ + return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], + Prefix: prefix, + }) + if err != nil { + t.Fatal(err) + } + wantCount := len(items) - startIndex + if count != wantCount { + t.Errorf("got %v items, want %v", count, wantCount) + } + }) + + t.Run("with prefix and skip start from", func(t *testing.T) { + startIndex := 2 + var count int + i := startIndex + 1 + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + count++ + return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], + SkipStartFromItem: true, + Prefix: prefix, + }) + if err != nil { + t.Fatal(err) + } + wantCount := len(items) - startIndex - 1 + if count != wantCount { + t.Errorf("got %v items, want %v", count, wantCount) + } + }) + + t.Run("stop", func(t *testing.T) { + var i int + stopIndex := 3 + var count int + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + count++ + if i == stopIndex { + return true, nil + } + i++ + return false, nil + }, &IterateOptions{ + Prefix: prefix, }) if err != nil { t.Fatal(err) @@ -369,46 +583,187 @@ func TestIndex_iterate(t *testing.T) { t.Fatal(err) } - secondIndexItem := IndexItem{ + secondItem := Item{ Address: []byte("iterate-hash-10"), Data: []byte("data-second"), } - err = secondIndex.Put(secondIndexItem) + err = secondIndex.Put(secondItem) if err != nil { t.Fatal(err) } var i int - err = index.IterateAll(func(item IndexItem) (stop bool, err error) { + err = index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } want := items[i] - checkIndexItem(t, item, want) + checkItem(t, item, want) i++ return false, nil + }, &IterateOptions{ + Prefix: prefix, }) if err != nil { t.Fatal(err) } - - i = 0 - err = secondIndex.IterateAll(func(item IndexItem) (stop bool, err error) { - if i > 1 { - return true, fmt.Errorf("got unexpected index item: %#v", item) - } - checkIndexItem(t, item, secondIndexItem) - i++ - return false, nil - }) - if err != nil { - t.Fatal(err) + if i != len(items) { + t.Errorf("got %v items, want %v", i, len(items)) } }) } -// checkIndexItem is a test helper function that compares if two Index items are the same. -func checkIndexItem(t *testing.T, got, want IndexItem) { +// TestIndex_count tests if Index.Count and Index.CountFrom +// returns the correct number of items. +func TestIndex_count(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + index, err := db.NewIndex("retrieval", retrievalIndexFuncs) + if err != nil { + t.Fatal(err) + } + + items := []Item{ + { + Address: []byte("iterate-hash-01"), + Data: []byte("data80"), + }, + { + Address: []byte("iterate-hash-02"), + Data: []byte("data84"), + }, + { + Address: []byte("iterate-hash-03"), + Data: []byte("data22"), + }, + { + Address: []byte("iterate-hash-04"), + Data: []byte("data41"), + }, + { + Address: []byte("iterate-hash-05"), + Data: []byte("data1"), + }, + } + batch := new(leveldb.Batch) + for _, i := range items { + index.PutInBatch(batch, i) + } + err = db.WriteBatch(batch) + if err != nil { + t.Fatal(err) + } + + t.Run("Count", func(t *testing.T) { + got, err := index.Count() + if err != nil { + t.Fatal(err) + } + + want := len(items) + if got != want { + t.Errorf("got %v items count, want %v", got, want) + } + }) + + t.Run("CountFrom", func(t *testing.T) { + got, err := index.CountFrom(Item{ + Address: items[1].Address, + }) + if err != nil { + t.Fatal(err) + } + + want := len(items) - 1 + if got != want { + t.Errorf("got %v items count, want %v", got, want) + } + }) + + // update the index with another item + t.Run("add item", func(t *testing.T) { + item04 := Item{ + Address: []byte("iterate-hash-06"), + Data: []byte("data0"), + } + err = index.Put(item04) + if err != nil { + t.Fatal(err) + } + + count := len(items) + 1 + + t.Run("Count", func(t *testing.T) { + got, err := index.Count() + if err != nil { + t.Fatal(err) + } + + want := count + if got != want { + t.Errorf("got %v items count, want %v", got, want) + } + }) + + t.Run("CountFrom", func(t *testing.T) { + got, err := index.CountFrom(Item{ + Address: items[1].Address, + }) + if err != nil { + t.Fatal(err) + } + + want := count - 1 + if got != want { + t.Errorf("got %v items count, want %v", got, want) + } + }) + }) + + // delete some items + t.Run("delete items", func(t *testing.T) { + deleteCount := 3 + + for _, item := range items[:deleteCount] { + err := index.Delete(item) + if err != nil { + t.Fatal(err) + } + } + + count := len(items) + 1 - deleteCount + + t.Run("Count", func(t *testing.T) { + got, err := index.Count() + if err != nil { + t.Fatal(err) + } + + want := count + if got != want { + t.Errorf("got %v items count, want %v", got, want) + } + }) + + t.Run("CountFrom", func(t *testing.T) { + got, err := index.CountFrom(Item{ + Address: items[deleteCount+1].Address, + }) + if err != nil { + t.Fatal(err) + } + + want := count - 1 + if got != want { + t.Errorf("got %v items count, want %v", got, want) + } + }) + }) +} + +// checkItem is a test helper function that compares if two Index items are the same. +func checkItem(t *testing.T, got, want Item) { t.Helper() if !bytes.Equal(got.Address, want.Address) { From 56a3f6c03cc3c7ae38ab7354f8615c014bb2102a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Mon, 7 Jan 2019 14:32:01 +0100 Subject: [PATCH 05/14] swarm/storage/mock/test: fix T.Fatal inside a goroutine (#18399) --- swarm/storage/mock/test/test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/swarm/storage/mock/test/test.go b/swarm/storage/mock/test/test.go index 10180985f3..69828b144f 100644 --- a/swarm/storage/mock/test/test.go +++ b/swarm/storage/mock/test/test.go @@ -196,17 +196,22 @@ func ImportExport(t *testing.T, outStore, inStore mock.GlobalStorer, n int) { r, w := io.Pipe() defer r.Close() + exportErrChan := make(chan error) go func() { defer w.Close() - if _, err := exporter.Export(w); err != nil { - t.Fatalf("export: %v", err) - } + + _, err := exporter.Export(w) + exportErrChan <- err }() if _, err := importer.Import(r); err != nil { t.Fatalf("import: %v", err) } + if err := <-exportErrChan; err != nil { + t.Fatalf("export: %v", err) + } + for i, addr := range addrs { chunkAddr := storage.Address(append(addr[:], []byte(strconv.FormatInt(int64(i)+1, 16))...)) data := []byte(strconv.FormatInt(int64(i)+1, 16)) From ae857e74bfda1f961dc5741441e5b36a2bb9aa93 Mon Sep 17 00:00:00 2001 From: holisticode Date: Mon, 7 Jan 2019 18:59:00 -0500 Subject: [PATCH 06/14] swarm, p2p/protocols: Stream accounting (#18337) * swarm: completed 1st phase of swap accounting * swarm, p2p/protocols: added stream pricing * swarm/network/stream: gofmt simplify stream.go * swarm: fixed review comments * swarm: used snapshots for swap tests * swarm: custom retrieve for swap (less cascaded requests at any one time) * swarm: addressed PR comments * swarm: log output formatting * swarm: removed parallelism in swap tests * swarm: swap tests simplification * swarm: removed swap_test.go * swarm/network/stream: added prefix space for comments * swarm/network/stream: unit test for prices * swarm/network/stream: don't hardcode price * swarm/network/stream: fixed invalid price check --- p2p/protocols/accounting.go | 148 +++++++++++++------------- swarm/network/stream/stream.go | 110 +++++++++++++------ swarm/network/stream/streamer_test.go | 31 ++++++ 3 files changed, 185 insertions(+), 104 deletions(-) diff --git a/p2p/protocols/accounting.go b/p2p/protocols/accounting.go index 770406a27e..bdc490e591 100644 --- a/p2p/protocols/accounting.go +++ b/p2p/protocols/accounting.go @@ -22,31 +22,33 @@ import ( "github.com/ethereum/go-ethereum/metrics" ) -//define some metrics +// define some metrics var ( - //All metrics are cumulative + // All metrics are cumulative - //total amount of units credited + // total amount of units credited mBalanceCredit metrics.Counter - //total amount of units debited + // total amount of units debited mBalanceDebit metrics.Counter - //total amount of bytes credited + // total amount of bytes credited mBytesCredit metrics.Counter - //total amount of bytes debited + // total amount of bytes debited mBytesDebit metrics.Counter - //total amount of credited messages + // total amount of credited messages mMsgCredit metrics.Counter - //total amount of debited messages + // total amount of debited messages mMsgDebit metrics.Counter - //how many times local node had to drop remote peers + // how many times local node had to drop remote peers mPeerDrops metrics.Counter - //how many times local node overdrafted and dropped + // how many times local node overdrafted and dropped mSelfDrops metrics.Counter + + MetricsRegistry metrics.Registry ) -//Prices defines how prices are being passed on to the accounting instance +// Prices defines how prices are being passed on to the accounting instance type Prices interface { - //Return the Price for a message + // Return the Price for a message Price(interface{}) *Price } @@ -57,20 +59,20 @@ const ( Receiver = Payer(false) ) -//Price represents the costs of a message +// Price represents the costs of a message type Price struct { - Value uint64 // - PerByte bool //True if the price is per byte or for unit + Value uint64 + PerByte bool // True if the price is per byte or for unit Payer Payer } -//For gives back the price for a message -//A protocol provides the message price in absolute value -//This method then returns the correct signed amount, -//depending on who pays, which is identified by the `payer` argument: -//`Send` will pass a `Sender` payer, `Receive` will pass the `Receiver` argument. -//Thus: If Sending and sender pays, amount positive, otherwise negative -//If Receiving, and receiver pays, amount positive, otherwise negative +// For gives back the price for a message +// A protocol provides the message price in absolute value +// This method then returns the correct signed amount, +// depending on who pays, which is identified by the `payer` argument: +// `Send` will pass a `Sender` payer, `Receive` will pass the `Receiver` argument. +// Thus: If Sending and sender pays, amount positive, otherwise negative +// If Receiving, and receiver pays, amount positive, otherwise negative func (p *Price) For(payer Payer, size uint32) int64 { price := p.Value if p.PerByte { @@ -82,22 +84,22 @@ func (p *Price) For(payer Payer, size uint32) int64 { return int64(price) } -//Balance is the actual accounting instance -//Balance defines the operations needed for accounting -//Implementations internally maintain the balance for every peer +// Balance is the actual accounting instance +// Balance defines the operations needed for accounting +// Implementations internally maintain the balance for every peer type Balance interface { - //Adds amount to the local balance with remote node `peer`; - //positive amount = credit local node - //negative amount = debit local node + // Adds amount to the local balance with remote node `peer`; + // positive amount = credit local node + // negative amount = debit local node Add(amount int64, peer *Peer) error } -//Accounting implements the Hook interface -//It interfaces to the balances through the Balance interface, -//while interfacing with protocols and its prices through the Prices interface +// Accounting implements the Hook interface +// It interfaces to the balances through the Balance interface, +// while interfacing with protocols and its prices through the Prices interface type Accounting struct { - Balance //interface to accounting logic - Prices //interface to prices logic + Balance // interface to accounting logic + Prices // interface to prices logic } func NewAccounting(balance Balance, po Prices) *Accounting { @@ -108,79 +110,77 @@ func NewAccounting(balance Balance, po Prices) *Accounting { return ah } -//SetupAccountingMetrics creates a separate registry for p2p accounting metrics; -//this registry should be independent of any other metrics as it persists at different endpoints. -//It also instantiates the given metrics and starts the persisting go-routine which -//at the passed interval writes the metrics to a LevelDB +// SetupAccountingMetrics creates a separate registry for p2p accounting metrics; +// this registry should be independent of any other metrics as it persists at different endpoints. +// It also instantiates the given metrics and starts the persisting go-routine which +// at the passed interval writes the metrics to a LevelDB func SetupAccountingMetrics(reportInterval time.Duration, path string) *AccountingMetrics { - //create an empty registry - registry := metrics.NewRegistry() - //instantiate the metrics - mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", registry) - mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", registry) - mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", registry) - mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", registry) - mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", registry) - mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", registry) - mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", registry) - mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", registry) - //create the DB and start persisting - return NewAccountingMetrics(registry, reportInterval, path) + // create an empty registry + MetricsRegistry = metrics.NewRegistry() + // instantiate the metrics + mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", MetricsRegistry) + mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", MetricsRegistry) + mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", MetricsRegistry) + mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", MetricsRegistry) + mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", MetricsRegistry) + mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", MetricsRegistry) + mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", MetricsRegistry) + mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", MetricsRegistry) + // create the DB and start persisting + return NewAccountingMetrics(MetricsRegistry, reportInterval, path) } -//Implement Hook.Send // Send takes a peer, a size and a msg and -// - calculates the cost for the local node sending a msg of size to peer using the Prices interface -// - credits/debits local node using balance interface +// - calculates the cost for the local node sending a msg of size to peer using the Prices interface +// - credits/debits local node using balance interface func (ah *Accounting) Send(peer *Peer, size uint32, msg interface{}) error { - //get the price for a message (through the protocol spec) + // get the price for a message (through the protocol spec) price := ah.Price(msg) - //this message doesn't need accounting + // this message doesn't need accounting if price == nil { return nil } - //evaluate the price for sending messages + // evaluate the price for sending messages costToLocalNode := price.For(Sender, size) - //do the accounting + // do the accounting err := ah.Add(costToLocalNode, peer) - //record metrics: just increase counters for user-facing metrics + // record metrics: just increase counters for user-facing metrics ah.doMetrics(costToLocalNode, size, err) return err } -//Implement Hook.Receive // Receive takes a peer, a size and a msg and -// - calculates the cost for the local node receiving a msg of size from peer using the Prices interface -// - credits/debits local node using balance interface +// - calculates the cost for the local node receiving a msg of size from peer using the Prices interface +// - credits/debits local node using balance interface func (ah *Accounting) Receive(peer *Peer, size uint32, msg interface{}) error { - //get the price for a message (through the protocol spec) + // get the price for a message (through the protocol spec) price := ah.Price(msg) - //this message doesn't need accounting + // this message doesn't need accounting if price == nil { return nil } - //evaluate the price for receiving messages + // evaluate the price for receiving messages costToLocalNode := price.For(Receiver, size) - //do the accounting + // do the accounting err := ah.Add(costToLocalNode, peer) - //record metrics: just increase counters for user-facing metrics + // record metrics: just increase counters for user-facing metrics ah.doMetrics(costToLocalNode, size, err) return err } -//record some metrics -//this is not an error handling. `err` is returned by both `Send` and `Receive` -//`err` will only be non-nil if a limit has been violated (overdraft), in which case the peer has been dropped. -//if the limit has been violated and `err` is thus not nil: -// * if the price is positive, local node has been credited; thus `err` implicitly signals the REMOTE has been dropped -// * if the price is negative, local node has been debited, thus `err` implicitly signals LOCAL node "overdraft" +// record some metrics +// this is not an error handling. `err` is returned by both `Send` and `Receive` +// `err` will only be non-nil if a limit has been violated (overdraft), in which case the peer has been dropped. +// if the limit has been violated and `err` is thus not nil: +// * if the price is positive, local node has been credited; thus `err` implicitly signals the REMOTE has been dropped +// * if the price is negative, local node has been debited, thus `err` implicitly signals LOCAL node "overdraft" func (ah *Accounting) doMetrics(price int64, size uint32, err error) { if price > 0 { mBalanceCredit.Inc(price) mBytesCredit.Inc(int64(size)) mMsgCredit.Inc(1) if err != nil { - //increase the number of times a remote node has been dropped due to "overdraft" + // increase the number of times a remote node has been dropped due to "overdraft" mPeerDrops.Inc(1) } } else { @@ -188,7 +188,7 @@ func (ah *Accounting) doMetrics(price int64, size uint32, err error) { mBytesDebit.Inc(int64(size)) mMsgDebit.Inc(1) if err != nil { - //increase the number of times the local node has done an "overdraft" in respect to other nodes + // increase the number of times the local node has done an "overdraft" in respect to other nodes mSelfDrops.Inc(1) } } diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 090bef8d17..2e2c3c418c 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -48,28 +48,28 @@ const ( HashSize = 32 ) -//Enumerate options for syncing and retrieval +// Enumerate options for syncing and retrieval type SyncingOption int type RetrievalOption int -//Syncing options +// Syncing options const ( - //Syncing disabled + // Syncing disabled SyncingDisabled SyncingOption = iota - //Register the client and the server but not subscribe + // Register the client and the server but not subscribe SyncingRegisterOnly - //Both client and server funcs are registered, subscribe sent automatically + // Both client and server funcs are registered, subscribe sent automatically SyncingAutoSubscribe ) const ( - //Retrieval disabled. Used mostly for tests to isolate syncing features (i.e. syncing only) + // Retrieval disabled. Used mostly for tests to isolate syncing features (i.e. syncing only) RetrievalDisabled RetrievalOption = iota - //Only the client side of the retrieve request is registered. - //(light nodes do not serve retrieve requests) - //once the client is registered, subscription to retrieve request stream is always sent + // Only the client side of the retrieve request is registered. + // (light nodes do not serve retrieve requests) + // once the client is registered, subscription to retrieve request stream is always sent RetrievalClientOnly - //Both client and server funcs are registered, subscribe sent automatically + // Both client and server funcs are registered, subscribe sent automatically RetrievalEnabled ) @@ -86,18 +86,18 @@ type Registry struct { peers map[enode.ID]*Peer delivery *Delivery intervalsStore state.Store - autoRetrieval bool //automatically subscribe to retrieve request stream + autoRetrieval bool // automatically subscribe to retrieve request stream maxPeerServers int - spec *protocols.Spec //this protocol's spec - balance protocols.Balance //implements protocols.Balance, for accounting - prices protocols.Prices //implements protocols.Prices, provides prices to accounting + balance protocols.Balance // implements protocols.Balance, for accounting + prices protocols.Prices // implements protocols.Prices, provides prices to accounting + spec *protocols.Spec // this protocol's spec } // RegistryOptions holds optional values for NewRegistry constructor. type RegistryOptions struct { SkipCheck bool - Syncing SyncingOption //Defines syncing behavior - Retrieval RetrievalOption //Defines retrieval behavior + Syncing SyncingOption // Defines syncing behavior + Retrieval RetrievalOption // Defines retrieval behavior SyncUpdateDelay time.Duration MaxPeerServers int // The limit of servers for each peer in registry } @@ -110,7 +110,7 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy if options.SyncUpdateDelay <= 0 { options.SyncUpdateDelay = 15 * time.Second } - //check if retriaval has been disabled + // check if retrieval has been disabled retrieval := options.Retrieval != RetrievalDisabled streamer := &Registry{ @@ -130,7 +130,7 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy streamer.api = NewAPI(streamer) delivery.getPeer = streamer.getPeer - //if retrieval is enabled, register the server func, so that retrieve requests will be served (non-light nodes only) + // if retrieval is enabled, register the server func, so that retrieve requests will be served (non-light nodes only) if options.Retrieval == RetrievalEnabled { streamer.RegisterServerFunc(swarmChunkServerStreamName, func(_ *Peer, _ string, live bool) (Server, error) { if !live { @@ -140,20 +140,20 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy }) } - //if retrieval is not disabled, register the client func (both light nodes and normal nodes can issue retrieve requests) + // if retrieval is not disabled, register the client func (both light nodes and normal nodes can issue retrieve requests) if options.Retrieval != RetrievalDisabled { streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, t string, live bool) (Client, error) { return NewSwarmSyncerClient(p, syncChunkStore, NewStream(swarmChunkServerStreamName, t, live)) }) } - //If syncing is not disabled, the syncing functions are registered (both client and server) + // If syncing is not disabled, the syncing functions are registered (both client and server) if options.Syncing != SyncingDisabled { RegisterSwarmSyncerServer(streamer, syncChunkStore) RegisterSwarmSyncerClient(streamer, syncChunkStore) } - //if syncing is set to automatically subscribe to the syncing stream, start the subscription process + // if syncing is set to automatically subscribe to the syncing stream, start the subscription process if options.Syncing == SyncingAutoSubscribe { // latestIntC function ensures that // - receiving from the in chan is not blocked by processing inside the for loop @@ -235,13 +235,17 @@ func NewRegistry(localID enode.ID, delivery *Delivery, syncChunkStore storage.Sy return streamer } -//we need to construct a spec instance per node instance +// This is an accounted protocol, therefore we need to provide a pricing Hook to the spec +// For simulations to be able to run multiple nodes and not override the hook's balance, +// we need to construct a spec instance per node instance func (r *Registry) setupSpec() { - //first create the "bare" spec + // first create the "bare" spec r.createSpec() - //if balance is nil, this node has been started without swap support (swapEnabled flag is false) + // now create the pricing object + r.createPriceOracle() + // if balance is nil, this node has been started without swap support (swapEnabled flag is false) if r.balance != nil && !reflect.ValueOf(r.balance).IsNil() { - //swap is enabled, so setup the hook + // swap is enabled, so setup the hook r.spec.Hook = protocols.NewAccounting(r.balance, r.prices) } } @@ -533,11 +537,11 @@ func (p *Peer) HandleMsg(ctx context.Context, msg interface{}) error { return p.handleWantedHashesMsg(ctx, msg) case *ChunkDeliveryMsgRetrieval: - //handling chunk delivery is the same for retrieval and syncing, so let's cast the msg + // handling chunk delivery is the same for retrieval and syncing, so let's cast the msg return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, ((*ChunkDeliveryMsg)(msg))) case *ChunkDeliveryMsgSyncing: - //handling chunk delivery is the same for retrieval and syncing, so let's cast the msg + // handling chunk delivery is the same for retrieval and syncing, so let's cast the msg return p.streamer.delivery.handleChunkDeliveryMsg(ctx, p, ((*ChunkDeliveryMsg)(msg))) case *RetrieveRequestMsg: @@ -726,9 +730,9 @@ func (c *clientParams) clientCreated() { close(c.clientCreatedC) } -//GetSpec returns the streamer spec to callers -//This used to be a global variable but for simulations with -//multiple nodes its fields (notably the Hook) would be overwritten +// GetSpec returns the streamer spec to callers +// This used to be a global variable but for simulations with +// multiple nodes its fields (notably the Hook) would be overwritten func (r *Registry) GetSpec() *protocols.Spec { return r.spec } @@ -756,6 +760,52 @@ func (r *Registry) createSpec() { r.spec = spec } +// An accountable message needs some meta information attached to it +// in order to evaluate the correct price +type StreamerPrices struct { + priceMatrix map[reflect.Type]*protocols.Price + registry *Registry +} + +// Price implements the accounting interface and returns the price for a specific message +func (sp *StreamerPrices) Price(msg interface{}) *protocols.Price { + t := reflect.TypeOf(msg).Elem() + return sp.priceMatrix[t] +} + +// Instead of hardcoding the price, get it +// through a function - it could be quite complex in the future +func (sp *StreamerPrices) getRetrieveRequestMsgPrice() uint64 { + return uint64(1) +} + +// Instead of hardcoding the price, get it +// through a function - it could be quite complex in the future +func (sp *StreamerPrices) getChunkDeliveryMsgRetrievalPrice() uint64 { + return uint64(1) +} + +// createPriceOracle sets up a matrix which can be queried to get +// the price for a message via the Price method +func (r *Registry) createPriceOracle() { + sp := &StreamerPrices{ + registry: r, + } + sp.priceMatrix = map[reflect.Type]*protocols.Price{ + reflect.TypeOf(ChunkDeliveryMsgRetrieval{}): { + Value: sp.getChunkDeliveryMsgRetrievalPrice(), // arbitrary price for now + PerByte: true, + Payer: protocols.Receiver, + }, + reflect.TypeOf(RetrieveRequestMsg{}): { + Value: sp.getRetrieveRequestMsgPrice(), // arbitrary price for now + PerByte: false, + Payer: protocols.Sender, + }, + } + r.prices = sp +} + func (r *Registry) Protocols() []p2p.Protocol { return []p2p.Protocol{ { diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go index 77fe55d340..e1b1c82860 100644 --- a/swarm/network/stream/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -921,3 +921,34 @@ func TestMaxPeerServersWithoutUnsubscribe(t *testing.T) { } } } + +//TestHasPriceImplementation is to check that the Registry has a +//`Price` interface implementation +func TestHasPriceImplementation(t *testing.T) { + _, r, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingDisabled, + }) + defer teardown() + if err != nil { + t.Fatal(err) + } + + if r.prices == nil { + t.Fatal("No prices implementation available for the stream protocol") + } + + pricesInstance, ok := r.prices.(*StreamerPrices) + if !ok { + t.Fatal("`Registry` does not have the expected Prices instance") + } + price := pricesInstance.Price(&ChunkDeliveryMsgRetrieval{}) + if price == nil || price.Value == 0 || price.Value != pricesInstance.getChunkDeliveryMsgRetrievalPrice() { + t.Fatal("No prices set for chunk delivery msg") + } + + price = pricesInstance.Price(&RetrieveRequestMsg{}) + if price == nil || price.Value == 0 || price.Value != pricesInstance.getRetrieveRequestMsgPrice() { + t.Fatal("No prices set for chunk delivery msg") + } +} From 81f04fa60608a67bac693879acbe086562d3970d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 8 Jan 2019 22:50:15 +0100 Subject: [PATCH 07/14] github: remove swarm github codeowners (#18412) --- .github/CODEOWNERS | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 11c4dcedc2..c03fa06c72 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,27 +10,4 @@ les/ @zsfelfoldi light/ @zsfelfoldi mobile/ @karalabe p2p/ @fjl @zsfelfoldi -p2p/simulations @lmars -p2p/protocols @zelig -swarm/api/http @justelad -swarm/bmt @zelig -swarm/dev @lmars -swarm/fuse @jmozah @holisticode -swarm/grafana_dashboards @nonsense -swarm/metrics @nonsense @holisticode -swarm/multihash @nolash -swarm/network/bitvector @zelig @janos -swarm/network/priorityqueue @zelig @janos -swarm/network/simulations @zelig @janos -swarm/network/stream @janos @zelig @holisticode @justelad -swarm/network/stream/intervals @janos -swarm/network/stream/testing @zelig -swarm/pot @zelig -swarm/pss @nolash @zelig @nonsense -swarm/services @zelig -swarm/state @justelad -swarm/storage/encryption @zelig @nagydani -swarm/storage/mock @janos -swarm/storage/feed @nolash @jpeletier -swarm/testutil @lmars whisper/ @gballet @gluk256 From d70c4faf20d5533e30eec5cbb9b5180eb837b78c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Wed, 9 Jan 2019 07:05:55 +0100 Subject: [PATCH 08/14] swarm: Fix T.Fatal inside a goroutine in tests (#18409) * swarm/storage: fix T.Fatal inside a goroutine * swarm/network/simulation: fix T.Fatal inside a goroutine * swarm/network/stream: fix T.Fatal inside a goroutine * swarm/network/simulation: consistent failures in TestPeerEventsTimeout * swarm/network/simulation: rename sendRunSignal to triggerSimulationRun --- swarm/network/simulation/events_test.go | 7 +- swarm/network/simulation/http_test.go | 21 ++-- swarm/network/stream/delivery_test.go | 34 ++++-- swarm/network/stream/intervals_test.go | 14 ++- swarm/network/stream/snapshot_sync_test.go | 23 +++- swarm/network/stream/syncer_test.go | 14 ++- swarm/storage/netstore_test.go | 122 ++++++++++++++------- 7 files changed, 167 insertions(+), 68 deletions(-) diff --git a/swarm/network/simulation/events_test.go b/swarm/network/simulation/events_test.go index 34ef24ed40..529844816f 100644 --- a/swarm/network/simulation/events_test.go +++ b/swarm/network/simulation/events_test.go @@ -81,6 +81,7 @@ func TestPeerEventsTimeout(t *testing.T) { events := sim.PeerEvents(ctx, sim.NodeIDs()) done := make(chan struct{}) + errC := make(chan error) go func() { for e := range events { if e.Error == context.Canceled { @@ -90,14 +91,16 @@ func TestPeerEventsTimeout(t *testing.T) { close(done) return } else { - t.Fatal(e.Error) + errC <- e.Error } } }() select { case <-time.After(time.Second): - t.Error("no context deadline received") + t.Fatal("no context deadline received") + case err := <-errC: + t.Fatal(err) case <-done: // all good, context deadline detected } diff --git a/swarm/network/simulation/http_test.go b/swarm/network/simulation/http_test.go index 775cf92190..dffd03a032 100644 --- a/swarm/network/simulation/http_test.go +++ b/swarm/network/simulation/http_test.go @@ -73,7 +73,8 @@ func TestSimulationWithHTTPServer(t *testing.T) { //this time the timeout should be long enough so that it doesn't kick in too early ctx, cancel2 := context.WithTimeout(context.Background(), 5*time.Second) defer cancel2() - go sendRunSignal(t) + errC := make(chan error, 1) + go triggerSimulationRun(t, errC) result = sim.Run(ctx, func(ctx context.Context, sim *Simulation) error { log.Debug("This run waits for the run signal from `frontend`...") //ensure with a Sleep that simulation doesn't terminate before the signal is received @@ -83,10 +84,13 @@ func TestSimulationWithHTTPServer(t *testing.T) { if result.Error != nil { t.Fatal(result.Error) } + if err := <-errC; err != nil { + t.Fatal(err) + } log.Debug("Test terminated successfully") } -func sendRunSignal(t *testing.T) { +func triggerSimulationRun(t *testing.T, errC chan error) { //We need to first wait for the sim HTTP server to start running... time.Sleep(2 * time.Second) //then we can send the signal @@ -94,16 +98,13 @@ func sendRunSignal(t *testing.T) { log.Debug("Sending run signal to simulation: POST /runsim...") resp, err := http.Post(fmt.Sprintf("http://localhost%s/runsim", DefaultHTTPSimAddr), "application/json", nil) if err != nil { - t.Fatalf("Request failed: %v", err) + errC <- fmt.Errorf("Request failed: %v", err) + return } - defer func() { - err := resp.Body.Close() - if err != nil { - log.Error("Error closing response body", "err", err) - } - }() log.Debug("Signal sent") if resp.StatusCode != http.StatusOK { - t.Fatalf("err %s", resp.Status) + errC <- fmt.Errorf("err %s", resp.Status) + return } + errC <- resp.Body.Close() } diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 04a4bb0779..c65e1386d0 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -19,9 +19,11 @@ package stream import ( "bytes" "context" + "errors" "fmt" "os" "sync" + "sync/atomic" "testing" "time" @@ -500,7 +502,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) log.Info("Starting simulation") ctx := context.Background() - result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) { nodeIDs := sim.UpNodeIDs() //determine the pivot node to be the first node of the simulation pivot := nodeIDs[0] @@ -553,14 +555,13 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) } pivotFileStore := item.(*storage.FileStore) log.Debug("Starting retrieval routine") + retErrC := make(chan error) go func() { // start the retrieval on the pivot node - this will spawn retrieve requests for missing chunks // we must wait for the peer connections to have started before requesting n, err := readAll(pivotFileStore, fileHash) log.Info(fmt.Sprintf("retrieved %v", fileHash), "read", n, "err", err) - if err != nil { - t.Fatalf("requesting chunks action error: %v", err) - } + retErrC <- err }() log.Debug("Watching for disconnections") @@ -570,11 +571,19 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) simulation.NewPeerEventsFilter().Drop(), ) + var disconnected atomic.Value go func() { for d := range disconnections { if d.Error != nil { log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - t.Fatal(d.Error) + disconnected.Store(true) + } + } + }() + defer func() { + if err != nil { + if yes, ok := disconnected.Load().(bool); ok && yes { + err = errors.New("disconnect events received") } } }() @@ -595,6 +604,9 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) if !success { return fmt.Errorf("Test failed, chunks not available on all nodes") } + if err := <-retErrC; err != nil { + t.Fatalf("requesting chunks: %v", err) + } log.Debug("Test terminated successfully") return nil }) @@ -675,7 +687,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b } ctx := context.Background() - result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) { nodeIDs := sim.UpNodeIDs() node := nodeIDs[len(nodeIDs)-1] @@ -702,11 +714,19 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b simulation.NewPeerEventsFilter().Drop(), ) + var disconnected atomic.Value go func() { for d := range disconnections { if d.Error != nil { log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - b.Fatal(d.Error) + disconnected.Store(true) + } + } + }() + defer func() { + if err != nil { + if yes, ok := disconnected.Load().(bool); ok && yes { + err = errors.New("disconnect events received") } } }() diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 7c7feeb112..121758c187 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -19,9 +19,11 @@ package stream import ( "context" "encoding/binary" + "errors" "fmt" "os" "sync" + "sync/atomic" "testing" "time" @@ -117,7 +119,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { t.Fatal(err) } - result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) { nodeIDs := sim.UpNodeIDs() storer := nodeIDs[0] checker := nodeIDs[1] @@ -162,11 +164,19 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { return err } + var disconnected atomic.Value go func() { for d := range disconnections { if d.Error != nil { log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - t.Fatal(d.Error) + disconnected.Store(true) + } + } + }() + defer func() { + if err != nil { + if yes, ok := disconnected.Load().(bool); ok && yes { + err = errors.New("disconnect events received") } } }() diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index f86d9accac..7a883644b9 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -21,6 +21,7 @@ import ( "os" "runtime" "sync" + "sync/atomic" "testing" "time" @@ -213,11 +214,13 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { simulation.NewPeerEventsFilter().Drop(), ) + var disconnected atomic.Value go func() { for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - t.Fatal("unexpected disconnect") - cancelSimRun() + if d.Error != nil { + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) + disconnected.Store(true) + } } }() @@ -226,6 +229,9 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { if result.Error != nil { t.Fatal(result.Error) } + if yes, ok := disconnected.Load().(bool); ok && yes { + t.Fatal("disconnect events received") + } log.Info("Simulation ended") } @@ -395,11 +401,13 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) simulation.NewPeerEventsFilter().Drop(), ) + var disconnected atomic.Value go func() { for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - t.Fatal("unexpected disconnect") - cancelSimRun() + if d.Error != nil { + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) + disconnected.Store(true) + } } }() @@ -514,6 +522,9 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) return result.Error } + if yes, ok := disconnected.Load().(bool); ok && yes { + t.Fatal("disconnect events received") + } log.Info("Simulation ended") return nil } diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 2fdba71ff1..09152ebfe1 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -18,11 +18,13 @@ package stream import ( "context" + "errors" "fmt" "io/ioutil" "math" "os" "sync" + "sync/atomic" "testing" "time" @@ -129,7 +131,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p if err != nil { t.Fatal(err) } - result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) { nodeIDs := sim.UpNodeIDs() nodeIndex := make(map[enode.ID]int) @@ -143,11 +145,19 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p simulation.NewPeerEventsFilter().Drop(), ) + var disconnected atomic.Value go func() { for d := range disconnections { if d.Error != nil { log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - t.Fatal(d.Error) + disconnected.Store(true) + } + } + }() + defer func() { + if err != nil { + if yes, ok := disconnected.Load().(bool); ok && yes { + err = errors.New("disconnect events received") } } }() diff --git a/swarm/storage/netstore_test.go b/swarm/storage/netstore_test.go index 8a09fa5ae0..2ed3e0752d 100644 --- a/swarm/storage/netstore_test.go +++ b/swarm/storage/netstore_test.go @@ -20,6 +20,8 @@ import ( "bytes" "context" "crypto/rand" + "errors" + "fmt" "io/ioutil" "sync" "testing" @@ -114,19 +116,24 @@ func TestNetStoreGetAndPut(t *testing.T) { defer cancel() c := make(chan struct{}) // this channel ensures that the gouroutine with the Put does not run earlier than the Get + putErrC := make(chan error) go func() { <-c // wait for the Get to be called time.Sleep(200 * time.Millisecond) // and a little more so it is surely called // check if netStore created a fetcher in the Get call for the unavailable chunk if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil { - t.Fatal("Expected netStore to use a fetcher for the Get call") + putErrC <- errors.New("Expected netStore to use a fetcher for the Get call") + return } err := netStore.Put(ctx, chunk) if err != nil { - t.Fatalf("Expected no err got %v", err) + putErrC <- fmt.Errorf("Expected no err got %v", err) + return } + + putErrC <- nil }() close(c) @@ -134,6 +141,10 @@ func TestNetStoreGetAndPut(t *testing.T) { if err != nil { t.Fatalf("Expected no err got %v", err) } + + if err := <-putErrC; err != nil { + t.Fatal(err) + } // the retrieved chunk should be the same as what we Put if !bytes.Equal(recChunk.Address(), chunk.Address()) || !bytes.Equal(recChunk.Data(), chunk.Data()) { t.Fatalf("Different chunk received than what was put") @@ -200,14 +211,18 @@ func TestNetStoreGetTimeout(t *testing.T) { defer cancel() c := make(chan struct{}) // this channel ensures that the gouroutine does not run earlier than the Get + fetcherErrC := make(chan error) go func() { <-c // wait for the Get to be called time.Sleep(200 * time.Millisecond) // and a little more so it is surely called // check if netStore created a fetcher in the Get call for the unavailable chunk if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil { - t.Fatal("Expected netStore to use a fetcher for the Get call") + fetcherErrC <- errors.New("Expected netStore to use a fetcher for the Get call") + return } + + fetcherErrC <- nil }() close(c) @@ -220,6 +235,10 @@ func TestNetStoreGetTimeout(t *testing.T) { t.Fatalf("Expected context.DeadLineExceeded err got %v", err) } + if err := <-fetcherErrC; err != nil { + t.Fatal(err) + } + // A fetcher was created, check if it has been removed after timeout if netStore.fetchers.Len() != 0 { t.Fatal("Expected netStore to remove the fetcher after timeout") @@ -243,20 +262,29 @@ func TestNetStoreGetCancel(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) c := make(chan struct{}) // this channel ensures that the gouroutine with the cancel does not run earlier than the Get + fetcherErrC := make(chan error, 1) go func() { <-c // wait for the Get to be called time.Sleep(200 * time.Millisecond) // and a little more so it is surely called // check if netStore created a fetcher in the Get call for the unavailable chunk if netStore.fetchers.Len() != 1 || netStore.getFetcher(chunk.Address()) == nil { - t.Fatal("Expected netStore to use a fetcher for the Get call") + fetcherErrC <- errors.New("Expected netStore to use a fetcher for the Get call") + return } + + fetcherErrC <- nil cancel() }() close(c) + // We call Get with an unavailable chunk, so it will create a fetcher and wait for delivery _, err := netStore.Get(ctx, chunk.Address()) + if err := <-fetcherErrC; err != nil { + t.Fatal(err) + } + // After the context is cancelled above Get should return with an error if err != context.Canceled { t.Fatalf("Expected context.Canceled err got %v", err) @@ -286,46 +314,55 @@ func TestNetStoreMultipleGetAndPut(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() + putErrC := make(chan error) go func() { // sleep to make sure Put is called after all the Get time.Sleep(500 * time.Millisecond) // check if netStore created exactly one fetcher for all Get calls if netStore.fetchers.Len() != 1 { - t.Fatal("Expected netStore to use one fetcher for all Get calls") + putErrC <- errors.New("Expected netStore to use one fetcher for all Get calls") + return } err := netStore.Put(ctx, chunk) if err != nil { - t.Fatalf("Expected no err got %v", err) + putErrC <- fmt.Errorf("Expected no err got %v", err) + return } + putErrC <- nil }() + count := 4 // call Get 4 times for the same unavailable chunk. The calls will be blocked until the Put above. - getWG := sync.WaitGroup{} - for i := 0; i < 4; i++ { - getWG.Add(1) + errC := make(chan error) + for i := 0; i < count; i++ { go func() { - defer getWG.Done() recChunk, err := netStore.Get(ctx, chunk.Address()) if err != nil { - t.Fatalf("Expected no err got %v", err) + errC <- fmt.Errorf("Expected no err got %v", err) } if !bytes.Equal(recChunk.Address(), chunk.Address()) || !bytes.Equal(recChunk.Data(), chunk.Data()) { - t.Fatalf("Different chunk received than what was put") + errC <- errors.New("Different chunk received than what was put") } + errC <- nil }() } - finishedC := make(chan struct{}) - go func() { - getWG.Wait() - close(finishedC) - }() + if err := <-putErrC; err != nil { + t.Fatal(err) + } + + timeout := time.After(1 * time.Second) // The Get calls should return after Put, so no timeout expected - select { - case <-finishedC: - case <-time.After(1 * time.Second): - t.Fatalf("Timeout waiting for Get calls to return") + for i := 0; i < count; i++ { + select { + case err := <-errC: + if err != nil { + t.Fatal(err) + } + case <-timeout: + t.Fatalf("Timeout waiting for Get calls to return") + } } // A fetcher was created, check if it has been removed after cancel @@ -448,7 +485,7 @@ func TestNetStoreGetCallsOffer(t *testing.T) { defer cancel() // We call get for a not available chunk, it will timeout because the chunk is not delivered - chunk, err := netStore.Get(ctx, chunk.Address()) + _, err := netStore.Get(ctx, chunk.Address()) if err != context.DeadlineExceeded { t.Fatalf("Expect error %v got %v", context.DeadlineExceeded, err) @@ -542,16 +579,12 @@ func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) { t.Fatalf("Expected netStore to have one fetcher for the requested chunk") } - // Call wait three times parallelly - wg := sync.WaitGroup{} - for i := 0; i < 3; i++ { - wg.Add(1) + // Call wait three times in parallel + count := 3 + errC := make(chan error) + for i := 0; i < count; i++ { go func() { - err := wait(ctx) - if err != nil { - t.Fatalf("Expected no err got %v", err) - } - wg.Done() + errC <- wait(ctx) }() } @@ -570,7 +603,12 @@ func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) { } // wait until all wait calls return (because the chunk is delivered) - wg.Wait() + for i := 0; i < count; i++ { + err := <-errC + if err != nil { + t.Fatal(err) + } + } // There should be no more fetchers for the delivered chunk if netStore.fetchers.Len() != 0 { @@ -606,23 +644,29 @@ func TestNetStoreFetcherLifeCycleWithTimeout(t *testing.T) { t.Fatalf("Expected netStore to have one fetcher for the requested chunk") } - // Call wait three times parallelly - wg := sync.WaitGroup{} - for i := 0; i < 3; i++ { - wg.Add(1) + // Call wait three times in parallel + count := 3 + errC := make(chan error) + for i := 0; i < count; i++ { go func() { - defer wg.Done() rctx, rcancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer rcancel() err := wait(rctx) if err != context.DeadlineExceeded { - t.Fatalf("Expected err %v got %v", context.DeadlineExceeded, err) + errC <- fmt.Errorf("Expected err %v got %v", context.DeadlineExceeded, err) + return } + errC <- nil }() } // wait until all wait calls timeout - wg.Wait() + for i := 0; i < count; i++ { + err := <-errC + if err != nil { + t.Fatal(err) + } + } // There should be no more fetchers after timeout if netStore.fetchers.Len() != 0 { From 6df3e4eeb0dda986aef3f71335151fa63c06f6d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Thu, 10 Jan 2019 03:36:19 +0100 Subject: [PATCH 09/14] swarm/network: remove isproxbin bool from kad.Each* iterfunc (#18239) * swarm/network, swarm/pss: remove isproxbin bool from kad.Each* iterfunc * swarm/network: restore comment and unskip snapshot sync tests --- swarm/network/discovery.go | 6 +++--- swarm/network/hive.go | 4 ++-- swarm/network/hive_test.go | 2 +- swarm/network/kademlia.go | 31 +++++++++++-------------------- swarm/network/kademlia_test.go | 2 +- swarm/network/networkid_test.go | 2 +- swarm/network/stream/delivery.go | 3 +-- swarm/network/stream/messages.go | 2 +- swarm/pss/pss.go | 2 +- swarm/pss/pss_test.go | 6 +++--- 10 files changed, 25 insertions(+), 35 deletions(-) diff --git a/swarm/network/discovery.go b/swarm/network/discovery.go index c6f5224301..4c503047a5 100644 --- a/swarm/network/discovery.go +++ b/swarm/network/discovery.go @@ -65,7 +65,7 @@ func (d *Peer) HandleMsg(ctx context.Context, msg interface{}) error { // NotifyDepth sends a message to all connections if depth of saturation is changed func NotifyDepth(depth uint8, kad *Kademlia) { - f := func(val *Peer, po int, _ bool) bool { + f := func(val *Peer, po int) bool { val.NotifyDepth(depth) return true } @@ -74,7 +74,7 @@ func NotifyDepth(depth uint8, kad *Kademlia) { // NotifyPeer informs all peers about a newly added node func NotifyPeer(p *BzzAddr, k *Kademlia) { - f := func(val *Peer, po int, _ bool) bool { + f := func(val *Peer, po int) bool { val.NotifyPeer(p, uint8(po)) return true } @@ -160,7 +160,7 @@ func (d *Peer) handleSubPeersMsg(msg *subPeersMsg) error { if !d.sentPeers { d.setDepth(msg.Depth) var peers []*BzzAddr - d.kad.EachConn(d.Over(), 255, func(p *Peer, po int, isproxbin bool) bool { + d.kad.EachConn(d.Over(), 255, func(p *Peer, po int) bool { if pob, _ := Pof(d, d.kad.BaseAddr(), 0); pob > po { return false } diff --git a/swarm/network/hive.go b/swarm/network/hive.go index ebef545929..a0b6b988ab 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -114,7 +114,7 @@ func (h *Hive) Stop() error { } } log.Info(fmt.Sprintf("%08x hive stopped, dropping peers", h.BaseAddr()[:4])) - h.EachConn(nil, 255, func(p *Peer, _ int, _ bool) bool { + h.EachConn(nil, 255, func(p *Peer, _ int) bool { log.Info(fmt.Sprintf("%08x dropping peer %08x", h.BaseAddr()[:4], p.Address()[:4])) p.Drop(nil) return true @@ -228,7 +228,7 @@ func (h *Hive) loadPeers() error { // savePeers, savePeer implement persistence callback/ func (h *Hive) savePeers() error { var peers []*BzzAddr - h.Kademlia.EachAddr(nil, 256, func(pa *BzzAddr, i int, _ bool) bool { + h.Kademlia.EachAddr(nil, 256, func(pa *BzzAddr, i int) bool { if pa == nil { log.Warn(fmt.Sprintf("empty addr: %v", i)) return true diff --git a/swarm/network/hive_test.go b/swarm/network/hive_test.go index 56adc5a8e9..a29e730833 100644 --- a/swarm/network/hive_test.go +++ b/swarm/network/hive_test.go @@ -103,7 +103,7 @@ func TestHiveStatePersistance(t *testing.T) { pp.Start(s1.Server) i := 0 - pp.Kademlia.EachAddr(nil, 256, func(addr *BzzAddr, po int, nn bool) bool { + pp.Kademlia.EachAddr(nil, 256, func(addr *BzzAddr, po int) bool { delete(peers, addr.String()) i++ return true diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index c5c2d79e3b..3214e151d1 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -390,46 +390,42 @@ func (k *Kademlia) EachBin(base []byte, pof pot.Pof, o int, eachBinFunc func(con // EachConn is an iterator with args (base, po, f) applies f to each live peer // that has proximity order po or less as measured from the base // if base is nil, kademlia base address is used -// It returns peers in order deepest to shallowest -func (k *Kademlia) EachConn(base []byte, o int, f func(*Peer, int, bool) bool) { +func (k *Kademlia) EachConn(base []byte, o int, f func(*Peer, int) bool) { k.lock.RLock() defer k.lock.RUnlock() k.eachConn(base, o, f) } -func (k *Kademlia) eachConn(base []byte, o int, f func(*Peer, int, bool) bool) { +func (k *Kademlia) eachConn(base []byte, o int, f func(*Peer, int) bool) { if len(base) == 0 { base = k.base } - depth := depthForPot(k.conns, k.MinProxBinSize, k.base) k.conns.EachNeighbour(base, Pof, func(val pot.Val, po int) bool { if po > o { return true } - return f(val.(*Peer), po, po >= depth) + return f(val.(*Peer), po) }) } // EachAddr called with (base, po, f) is an iterator applying f to each known peer // that has proximity order o or less as measured from the base // if base is nil, kademlia base address is used -// It returns peers in order deepest to shallowest -func (k *Kademlia) EachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool) { +func (k *Kademlia) EachAddr(base []byte, o int, f func(*BzzAddr, int) bool) { k.lock.RLock() defer k.lock.RUnlock() k.eachAddr(base, o, f) } -func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int, bool) bool) { +func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int) bool) { if len(base) == 0 { base = k.base } - depth := depthForPot(k.conns, k.MinProxBinSize, k.base) k.addrs.EachNeighbour(base, Pof, func(val pot.Val, po int) bool { if po > o { return true } - return f(val.(*entry).BzzAddr, po, po >= depth) + return f(val.(*entry).BzzAddr, po) }) } @@ -687,12 +683,11 @@ func (k *Kademlia) saturation() int { // TODO move to separate testing tools file func (k *Kademlia) knowNeighbours(addrs [][]byte) (got bool, n int, missing [][]byte) { pm := make(map[string]bool) - + depth := depthForPot(k.conns, k.MinProxBinSize, k.base) // create a map with all peers at depth and deeper known in the kademlia - // in order deepest to shallowest compared to the kademlia base address - // all bins (except self) are included (0 <= bin <= 255) - depth := depthForPot(k.addrs, k.MinProxBinSize, k.base) - k.eachAddr(nil, 255, func(p *BzzAddr, po int, nn bool) bool { + k.eachAddr(nil, 255, func(p *BzzAddr, po int) bool { + // in order deepest to shallowest compared to the kademlia base address + // all bins (except self) are included (0 <= bin <= 255) if po < depth { return false } @@ -724,12 +719,8 @@ func (k *Kademlia) knowNeighbours(addrs [][]byte) (got bool, n int, missing [][] // It is used in Healthy function for testing only func (k *Kademlia) connectedNeighbours(peers [][]byte) (got bool, n int, missing [][]byte) { pm := make(map[string]bool) - - // create a map with all peers at depth and deeper that are connected in the kademlia - // in order deepest to shallowest compared to the kademlia base address - // all bins (except self) are included (0 <= bin <= 255) depth := depthForPot(k.conns, k.MinProxBinSize, k.base) - k.eachConn(nil, 255, func(p *Peer, po int, nn bool) bool { + k.eachConn(nil, 255, func(p *Peer, po int) bool { if po < depth { return false } diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 773f201ac0..8d320c1728 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -232,7 +232,7 @@ func assertHealth(t *testing.T, k *Kademlia, expectHealthy bool, expectSaturatio t.Helper() kid := common.Bytes2Hex(k.BaseAddr()) addrs := [][]byte{k.BaseAddr()} - k.EachAddr(nil, 255, func(addr *BzzAddr, po int, _ bool) bool { + k.EachAddr(nil, 255, func(addr *BzzAddr, po int) bool { addrs = append(addrs, addr.Address()) return true }) diff --git a/swarm/network/networkid_test.go b/swarm/network/networkid_test.go index 191d67e5b6..3cd683e1c9 100644 --- a/swarm/network/networkid_test.go +++ b/swarm/network/networkid_test.go @@ -92,7 +92,7 @@ func TestNetworkID(t *testing.T) { if kademlias[node].addrs.Size() != len(netIDGroup)-1 { t.Fatalf("Kademlia size has not expected peer size. Kademlia size: %d, expected size: %d", kademlias[node].addrs.Size(), len(netIDGroup)-1) } - kademlias[node].EachAddr(nil, 0, func(addr *BzzAddr, _ int, _ bool) bool { + kademlias[node].EachAddr(nil, 0, func(addr *BzzAddr, _ int) bool { found := false for _, nd := range netIDGroup { if bytes.Equal(kademlias[nd].BaseAddr(), addr.Address()) { diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index c73298d9aa..e1a13fe8d5 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -19,7 +19,6 @@ package stream import ( "context" "errors" - "fmt" "github.com/ethereum/go-ethereum/metrics" @@ -245,7 +244,7 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) ( return nil, nil, fmt.Errorf("source peer %v not found", spID.String()) } } else { - d.kad.EachConn(req.Addr[:], 255, func(p *network.Peer, po int, nn bool) bool { + d.kad.EachConn(req.Addr[:], 255, func(p *network.Peer, po int) bool { id := p.ID() if p.LightNode { // skip light nodes diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index eb1b2983e1..b293724cc7 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -336,7 +336,7 @@ func (p *Peer) handleWantedHashesMsg(ctx context.Context, req *WantedHashesMsg) // launch in go routine since GetBatch blocks until new hashes arrive go func() { if err := p.SendOfferedHashes(s, req.From, req.To); err != nil { - log.Warn("SendOfferedHashes error", "err", err) + log.Warn("SendOfferedHashes error", "peer", p.ID().TerminalString(), "err", err) } }() // go p.SendOfferedHashes(s, req.From, req.To) diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 631d27f095..bee64b0df5 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -964,7 +964,7 @@ func (p *Pss) forward(msg *PssMsg) error { onlySendOnce = true } - p.Kademlia.EachConn(to, addressLength*8, func(sp *network.Peer, po int, _ bool) bool { + p.Kademlia.EachConn(to, addressLength*8, func(sp *network.Peer, po int) bool { if po < broadcastThreshold && sent > 0 { return false // stop iterating } diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index ec46504c23..b0753ad178 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -491,12 +491,12 @@ func TestAddressMatchProx(t *testing.T) { // meanwhile test regression for kademlia since we are compiling the test parameters from different packages var proxes int var conns int - kad.EachConn(nil, peerCount, func(p *network.Peer, po int, prox bool) bool { + depth := kad.NeighbourhoodDepth() + kad.EachConn(nil, peerCount, func(p *network.Peer, po int) bool { conns++ - if prox { + if po >= depth { proxes++ } - log.Trace("kadconn", "po", po, "peer", p, "prox", prox) return true }) if proxes != nnPeerCount { From 7ca40306af9da68a0d31439117246de8247f99d6 Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 10 Jan 2019 16:59:37 +0800 Subject: [PATCH 10/14] accounts/abi: tuple support (#18406) --- accounts/abi/abi.go | 2 - accounts/abi/abi_test.go | 57 +++-- accounts/abi/argument.go | 187 ++++++++++------ accounts/abi/pack_test.go | 302 ++++++++++++++++++++++++- accounts/abi/reflect.go | 63 +++--- accounts/abi/type.go | 141 ++++++++++-- accounts/abi/type_test.go | 433 +++++++++++++++++++----------------- accounts/abi/unpack.go | 92 +++++--- accounts/abi/unpack_test.go | 119 +++++++++- 9 files changed, 1003 insertions(+), 393 deletions(-) diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index 4d29814b21..08d5db9798 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -58,13 +58,11 @@ func (abi ABI) Pack(name string, args ...interface{}) ([]byte, error) { return nil, err } return arguments, nil - } method, exist := abi.Methods[name] if !exist { return nil, fmt.Errorf("method '%s' not found", name) } - arguments, err := method.Inputs.Pack(args...) if err != nil { return nil, err diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index 59ba79cb6d..b9444f9f0d 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -22,11 +22,10 @@ import ( "fmt" "log" "math/big" + "reflect" "strings" "testing" - "reflect" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" ) @@ -52,11 +51,14 @@ const jsondata2 = ` { "type" : "function", "name" : "slice", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint32[2]" } ] }, { "type" : "function", "name" : "slice256", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "uint256[2]" } ] }, { "type" : "function", "name" : "sliceAddress", "constant" : false, "inputs" : [ { "name" : "inputs", "type" : "address[]" } ] }, - { "type" : "function", "name" : "sliceMultiAddress", "constant" : false, "inputs" : [ { "name" : "a", "type" : "address[]" }, { "name" : "b", "type" : "address[]" } ] } + { "type" : "function", "name" : "sliceMultiAddress", "constant" : false, "inputs" : [ { "name" : "a", "type" : "address[]" }, { "name" : "b", "type" : "address[]" } ] }, + { "type" : "function", "name" : "nestedArray", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint256[2][2]" }, { "name" : "b", "type" : "address[]" } ] }, + { "type" : "function", "name" : "nestedArray2", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint8[][2]" } ] }, + { "type" : "function", "name" : "nestedSlice", "constant" : false, "inputs" : [ { "name" : "a", "type" : "uint8[][]" } ] } ]` func TestReader(t *testing.T) { - Uint256, _ := NewType("uint256") + Uint256, _ := NewType("uint256", nil) exp := ABI{ Methods: map[string]Method{ "balance": { @@ -177,7 +179,7 @@ func TestTestSlice(t *testing.T) { } func TestMethodSignature(t *testing.T) { - String, _ := NewType("string") + String, _ := NewType("string", nil) m := Method{"foo", false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil} exp := "foo(string,string)" if m.Sig() != exp { @@ -189,12 +191,31 @@ func TestMethodSignature(t *testing.T) { t.Errorf("expected ids to match %x != %x", m.Id(), idexp) } - uintt, _ := NewType("uint256") + uintt, _ := NewType("uint256", nil) m = Method{"foo", false, []Argument{{"bar", uintt, false}}, nil} exp = "foo(uint256)" if m.Sig() != exp { t.Error("signature mismatch", exp, "!=", m.Sig()) } + + // Method with tuple arguments + s, _ := NewType("tuple", []ArgumentMarshaling{ + {Name: "a", Type: "int256"}, + {Name: "b", Type: "int256[]"}, + {Name: "c", Type: "tuple[]", Components: []ArgumentMarshaling{ + {Name: "x", Type: "int256"}, + {Name: "y", Type: "int256"}, + }}, + {Name: "d", Type: "tuple[2]", Components: []ArgumentMarshaling{ + {Name: "x", Type: "int256"}, + {Name: "y", Type: "int256"}, + }}, + }) + m = Method{"foo", false, []Argument{{"s", s, false}, {"bar", String, false}}, nil} + exp = "foo((int256,int256[],(int256,int256)[],(int256,int256)[2]),string)" + if m.Sig() != exp { + t.Error("signature mismatch", exp, "!=", m.Sig()) + } } func TestMultiPack(t *testing.T) { @@ -564,11 +585,13 @@ func TestBareEvents(t *testing.T) { const definition = `[ { "type" : "event", "name" : "balance" }, { "type" : "event", "name" : "anon", "anonymous" : true}, - { "type" : "event", "name" : "args", "inputs" : [{ "indexed":false, "name":"arg0", "type":"uint256" }, { "indexed":true, "name":"arg1", "type":"address" }] } + { "type" : "event", "name" : "args", "inputs" : [{ "indexed":false, "name":"arg0", "type":"uint256" }, { "indexed":true, "name":"arg1", "type":"address" }] }, + { "type" : "event", "name" : "tuple", "inputs" : [{ "indexed":false, "name":"t", "type":"tuple", "components":[{"name":"a", "type":"uint256"}] }, { "indexed":true, "name":"arg1", "type":"address" }] } ]` - arg0, _ := NewType("uint256") - arg1, _ := NewType("address") + arg0, _ := NewType("uint256", nil) + arg1, _ := NewType("address", nil) + tuple, _ := NewType("tuple", []ArgumentMarshaling{{Name: "a", Type: "uint256"}}) expectedEvents := map[string]struct { Anonymous bool @@ -580,6 +603,10 @@ func TestBareEvents(t *testing.T) { {Name: "arg0", Type: arg0, Indexed: false}, {Name: "arg1", Type: arg1, Indexed: true}, }}, + "tuple": {false, []Argument{ + {Name: "t", Type: tuple, Indexed: false}, + {Name: "arg1", Type: arg1, Indexed: true}, + }}, } abi, err := JSON(strings.NewReader(definition)) @@ -646,28 +673,24 @@ func TestUnpackEvent(t *testing.T) { } type ReceivedEvent struct { - Address common.Address - Amount *big.Int - Memo []byte + Sender common.Address + Amount *big.Int + Memo []byte } var ev ReceivedEvent err = abi.Unpack(&ev, "received", data) if err != nil { t.Error(err) - } else { - t.Logf("len(data): %d; received event: %+v", len(data), ev) } type ReceivedAddrEvent struct { - Address common.Address + Sender common.Address } var receivedAddrEv ReceivedAddrEvent err = abi.Unpack(&receivedAddrEv, "receivedAddr", data) if err != nil { t.Error(err) - } else { - t.Logf("len(data): %d; received event: %+v", len(data), receivedAddrEv) } } diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index fdc6ff1648..d0a6b035c6 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -33,24 +33,27 @@ type Argument struct { type Arguments []Argument +type ArgumentMarshaling struct { + Name string + Type string + Components []ArgumentMarshaling + Indexed bool +} + // UnmarshalJSON implements json.Unmarshaler interface func (argument *Argument) UnmarshalJSON(data []byte) error { - var extarg struct { - Name string - Type string - Indexed bool - } - err := json.Unmarshal(data, &extarg) + var arg ArgumentMarshaling + err := json.Unmarshal(data, &arg) if err != nil { return fmt.Errorf("argument json err: %v", err) } - argument.Type, err = NewType(extarg.Type) + argument.Type, err = NewType(arg.Type, arg.Components) if err != nil { return err } - argument.Name = extarg.Name - argument.Indexed = extarg.Indexed + argument.Name = arg.Name + argument.Indexed = arg.Indexed return nil } @@ -85,7 +88,6 @@ func (arguments Arguments) isTuple() bool { // Unpack performs the operation hexdata -> Go format func (arguments Arguments) Unpack(v interface{}, data []byte) error { - // make sure the passed value is arguments pointer if reflect.Ptr != reflect.ValueOf(v).Kind() { return fmt.Errorf("abi: Unpack(non-pointer %T)", v) @@ -97,52 +99,134 @@ func (arguments Arguments) Unpack(v interface{}, data []byte) error { if arguments.isTuple() { return arguments.unpackTuple(v, marshalledValues) } - return arguments.unpackAtomic(v, marshalledValues) + return arguments.unpackAtomic(v, marshalledValues[0]) } -func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interface{}) error { +// unpack sets the unmarshalled value to go format. +// Note the dst here must be settable. +func unpack(t *Type, dst interface{}, src interface{}) error { + var ( + dstVal = reflect.ValueOf(dst).Elem() + srcVal = reflect.ValueOf(src) + ) + if t.T != TupleTy && !((t.T == SliceTy || t.T == ArrayTy) && t.Elem.T == TupleTy) { + return set(dstVal, srcVal) + } + + switch t.T { + case TupleTy: + if dstVal.Kind() != reflect.Struct { + return fmt.Errorf("abi: invalid dst value for unpack, want struct, got %s", dstVal.Kind()) + } + fieldmap, err := mapArgNamesToStructFields(t.TupleRawNames, dstVal) + if err != nil { + return err + } + for i, elem := range t.TupleElems { + fname := fieldmap[t.TupleRawNames[i]] + field := dstVal.FieldByName(fname) + if !field.IsValid() { + return fmt.Errorf("abi: field %s can't found in the given value", t.TupleRawNames[i]) + } + if err := unpack(elem, field.Addr().Interface(), srcVal.Field(i).Interface()); err != nil { + return err + } + } + return nil + case SliceTy: + if dstVal.Kind() != reflect.Slice { + return fmt.Errorf("abi: invalid dst value for unpack, want slice, got %s", dstVal.Kind()) + } + slice := reflect.MakeSlice(dstVal.Type(), srcVal.Len(), srcVal.Len()) + for i := 0; i < slice.Len(); i++ { + if err := unpack(t.Elem, slice.Index(i).Addr().Interface(), srcVal.Index(i).Interface()); err != nil { + return err + } + } + dstVal.Set(slice) + case ArrayTy: + if dstVal.Kind() != reflect.Array { + return fmt.Errorf("abi: invalid dst value for unpack, want array, got %s", dstVal.Kind()) + } + array := reflect.New(dstVal.Type()).Elem() + for i := 0; i < array.Len(); i++ { + if err := unpack(t.Elem, array.Index(i).Addr().Interface(), srcVal.Index(i).Interface()); err != nil { + return err + } + } + dstVal.Set(array) + } + return nil +} + +// unpackAtomic unpacks ( hexdata -> go ) a single value +func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues interface{}) error { + if arguments.LengthNonIndexed() == 0 { + return nil + } + argument := arguments.NonIndexed()[0] + elem := reflect.ValueOf(v).Elem() + + if elem.Kind() == reflect.Struct { + fieldmap, err := mapArgNamesToStructFields([]string{argument.Name}, elem) + if err != nil { + return err + } + field := elem.FieldByName(fieldmap[argument.Name]) + if !field.IsValid() { + return fmt.Errorf("abi: field %s can't be found in the given value", argument.Name) + } + return unpack(&argument.Type, field.Addr().Interface(), marshalledValues) + } + return unpack(&argument.Type, elem.Addr().Interface(), marshalledValues) +} + +// unpackTuple unpacks ( hexdata -> go ) a batch of values. +func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interface{}) error { var ( value = reflect.ValueOf(v).Elem() typ = value.Type() kind = value.Kind() ) - if err := requireUnpackKind(value, typ, kind, arguments); err != nil { return err } // If the interface is a struct, get of abi->struct_field mapping - var abi2struct map[string]string if kind == reflect.Struct { - var err error - abi2struct, err = mapAbiToStructFields(arguments, value) + var ( + argNames []string + err error + ) + for _, arg := range arguments.NonIndexed() { + argNames = append(argNames, arg.Name) + } + abi2struct, err = mapArgNamesToStructFields(argNames, value) if err != nil { return err } } for i, arg := range arguments.NonIndexed() { - - reflectValue := reflect.ValueOf(marshalledValues[i]) - switch kind { case reflect.Struct: - if structField, ok := abi2struct[arg.Name]; ok { - if err := set(value.FieldByName(structField), reflectValue, arg); err != nil { - return err - } + field := value.FieldByName(abi2struct[arg.Name]) + if !field.IsValid() { + return fmt.Errorf("abi: field %s can't be found in the given value", arg.Name) + } + if err := unpack(&arg.Type, field.Addr().Interface(), marshalledValues[i]); err != nil { + return err } case reflect.Slice, reflect.Array: if value.Len() < i { return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(arguments), value.Len()) } v := value.Index(i) - if err := requireAssignable(v, reflectValue); err != nil { + if err := requireAssignable(v, reflect.ValueOf(marshalledValues[i])); err != nil { return err } - - if err := set(v.Elem(), reflectValue, arg); err != nil { + if err := unpack(&arg.Type, v.Addr().Interface(), marshalledValues[i]); err != nil { return err } default: @@ -150,48 +234,7 @@ func (arguments Arguments) unpackTuple(v interface{}, marshalledValues []interfa } } return nil -} -// unpackAtomic unpacks ( hexdata -> go ) a single value -func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interface{}) error { - if len(marshalledValues) != 1 { - return fmt.Errorf("abi: wrong length, expected single value, got %d", len(marshalledValues)) - } - - elem := reflect.ValueOf(v).Elem() - kind := elem.Kind() - reflectValue := reflect.ValueOf(marshalledValues[0]) - - var abi2struct map[string]string - if kind == reflect.Struct { - var err error - if abi2struct, err = mapAbiToStructFields(arguments, elem); err != nil { - return err - } - arg := arguments.NonIndexed()[0] - if structField, ok := abi2struct[arg.Name]; ok { - return set(elem.FieldByName(structField), reflectValue, arg) - } - return nil - } - - return set(elem, reflectValue, arguments.NonIndexed()[0]) - -} - -// Computes the full size of an array; -// i.e. counting nested arrays, which count towards size for unpacking. -func getArraySize(arr *Type) int { - size := arr.Size - // Arrays can be nested, with each element being the same size - arr = arr.Elem - for arr.T == ArrayTy { - // Keep multiplying by elem.Size while the elem is an array. - size *= arr.Size - arr = arr.Elem - } - // Now we have the full array size, including its children. - return size } // UnpackValues can be used to unpack ABI-encoded hexdata according to the ABI-specification, @@ -202,7 +245,7 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) { virtualArgs := 0 for index, arg := range arguments.NonIndexed() { marshalledValue, err := toGoType((index+virtualArgs)*32, arg.Type, data) - if arg.Type.T == ArrayTy && (*arg.Type.Elem).T != StringTy { + if arg.Type.T == ArrayTy && !isDynamicType(arg.Type) { // If we have a static array, like [3]uint256, these are coded as // just like uint256,uint256,uint256. // This means that we need to add two 'virtual' arguments when @@ -213,7 +256,11 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) { // // Calculate the full array size to get the correct offset for the next argument. // Decrement it by 1, as the normal index increment is still applied. - virtualArgs += getArraySize(&arg.Type) - 1 + virtualArgs += getTypeSize(arg.Type)/32 - 1 + } else if arg.Type.T == TupleTy && !isDynamicType(arg.Type) { + // If we have a static tuple, like (uint256, bool, uint256), these are + // coded as just like uint256,bool,uint256 + virtualArgs += getTypeSize(arg.Type)/32 - 1 } if err != nil { return nil, err @@ -243,7 +290,7 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { // input offset is the bytes offset for packed output inputOffset := 0 for _, abiArg := range abiArgs { - inputOffset += getDynamicTypeOffset(abiArg.Type) + inputOffset += getTypeSize(abiArg.Type) } var ret []byte for i, a := range args { diff --git a/accounts/abi/pack_test.go b/accounts/abi/pack_test.go index ddd2b7362d..10cd3a3962 100644 --- a/accounts/abi/pack_test.go +++ b/accounts/abi/pack_test.go @@ -29,303 +29,356 @@ import ( func TestPack(t *testing.T) { for i, test := range []struct { - typ string - - input interface{} - output []byte + typ string + components []ArgumentMarshaling + input interface{} + output []byte }{ { "uint8", + nil, uint8(2), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), }, { "uint8[]", + nil, []uint8{1, 2}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), }, { "uint16", + nil, uint16(2), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), }, { "uint16[]", + nil, []uint16{1, 2}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), }, { "uint32", + nil, uint32(2), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), }, { "uint32[]", + nil, []uint32{1, 2}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), }, { "uint64", + nil, uint64(2), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), }, { "uint64[]", + nil, []uint64{1, 2}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), }, { "uint256", + nil, big.NewInt(2), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), }, { "uint256[]", + nil, []*big.Int{big.NewInt(1), big.NewInt(2)}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), }, { "int8", + nil, int8(2), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), }, { "int8[]", + nil, []int8{1, 2}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), }, { "int16", + nil, int16(2), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), }, { "int16[]", + nil, []int16{1, 2}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), }, { "int32", + nil, int32(2), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), }, { "int32[]", + nil, []int32{1, 2}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), }, { "int64", + nil, int64(2), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), }, { "int64[]", + nil, []int64{1, 2}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), }, { "int256", + nil, big.NewInt(2), common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002"), }, { "int256[]", + nil, []*big.Int{big.NewInt(1), big.NewInt(2)}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002"), }, { "bytes1", + nil, [1]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes2", + nil, [2]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes3", + nil, [3]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes4", + nil, [4]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes5", + nil, [5]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes6", + nil, [6]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes7", + nil, [7]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes8", + nil, [8]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes9", + nil, [9]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes10", + nil, [10]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes11", + nil, [11]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes12", + nil, [12]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes13", + nil, [13]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes14", + nil, [14]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes15", + nil, [15]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes16", + nil, [16]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes17", + nil, [17]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes18", + nil, [18]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes19", + nil, [19]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes20", + nil, [20]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes21", + nil, [21]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes22", + nil, [22]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes23", + nil, [23]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes24", - [24]byte{1}, - common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), - }, - { - "bytes24", + nil, [24]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes25", + nil, [25]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes26", + nil, [26]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes27", + nil, [27]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes28", + nil, [28]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes29", + nil, [29]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes30", + nil, [30]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes31", + nil, [31]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "bytes32", + nil, [32]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "uint32[2][3][4]", + nil, [4][3][2]uint32{{{1, 2}, {3, 4}, {5, 6}}, {{7, 8}, {9, 10}, {11, 12}}, {{13, 14}, {15, 16}, {17, 18}}, {{19, 20}, {21, 22}, {23, 24}}}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000b000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000f000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000001300000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000000000000000000000000000015000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000170000000000000000000000000000000000000000000000000000000000000018"), }, { "address[]", + nil, []common.Address{{1}, {2}}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000200000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000"), }, { "bytes32[]", + nil, []common.Hash{{1}, {2}}, common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000201000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000"), }, { "function", + nil, [24]byte{1}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"), }, { "string", + nil, "foobar", common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006666f6f6261720000000000000000000000000000000000000000000000000000"), }, { "string[]", + nil, []string{"hello", "foobar"}, common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2 "0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 @@ -337,6 +390,7 @@ func TestPack(t *testing.T) { }, { "string[2]", + nil, []string{"hello", "foobar"}, common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset to i = 0 "0000000000000000000000000000000000000000000000000000000000000080" + // offset to i = 1 @@ -347,6 +401,7 @@ func TestPack(t *testing.T) { }, { "bytes32[][]", + nil, [][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}}, common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2 "0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 @@ -362,6 +417,7 @@ func TestPack(t *testing.T) { { "bytes32[][2]", + nil, [][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}}, common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 "00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1 @@ -376,6 +432,7 @@ func TestPack(t *testing.T) { { "bytes32[3][2]", + nil, [][]common.Hash{{{1}, {2}, {3}}, {{3}, {4}, {5}}}, common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0] "0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1] @@ -384,12 +441,182 @@ func TestPack(t *testing.T) { "0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1] "0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2] }, + { + // static tuple + "tuple", + []ArgumentMarshaling{ + {Name: "a", Type: "int64"}, + {Name: "b", Type: "int256"}, + {Name: "c", Type: "int256"}, + {Name: "d", Type: "bool"}, + {Name: "e", Type: "bytes32[3][2]"}, + }, + struct { + A int64 + B *big.Int + C *big.Int + D bool + E [][]common.Hash + }{1, big.NewInt(1), big.NewInt(-1), true, [][]common.Hash{{{1}, {2}, {3}}, {{3}, {4}, {5}}}}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001" + // struct[a] + "0000000000000000000000000000000000000000000000000000000000000001" + // struct[b] + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + // struct[c] + "0000000000000000000000000000000000000000000000000000000000000001" + // struct[d] + "0100000000000000000000000000000000000000000000000000000000000000" + // struct[e] array[0][0] + "0200000000000000000000000000000000000000000000000000000000000000" + // struct[e] array[0][1] + "0300000000000000000000000000000000000000000000000000000000000000" + // struct[e] array[0][2] + "0300000000000000000000000000000000000000000000000000000000000000" + // struct[e] array[1][0] + "0400000000000000000000000000000000000000000000000000000000000000" + // struct[e] array[1][1] + "0500000000000000000000000000000000000000000000000000000000000000"), // struct[e] array[1][2] + }, + { + // dynamic tuple + "tuple", + []ArgumentMarshaling{ + {Name: "a", Type: "string"}, + {Name: "b", Type: "int64"}, + {Name: "c", Type: "bytes"}, + {Name: "d", Type: "string[]"}, + {Name: "e", Type: "int256[]"}, + {Name: "f", Type: "address[]"}, + }, + struct { + FieldA string `abi:"a"` // Test whether abi tag works + FieldB int64 `abi:"b"` + C []byte + D []string + E []*big.Int + F []common.Address + }{"foobar", 1, []byte{1}, []string{"foo", "bar"}, []*big.Int{big.NewInt(1), big.NewInt(-1)}, []common.Address{{1}, {2}}}, + common.Hex2Bytes("00000000000000000000000000000000000000000000000000000000000000c0" + // struct[a] offset + "0000000000000000000000000000000000000000000000000000000000000001" + // struct[b] + "0000000000000000000000000000000000000000000000000000000000000100" + // struct[c] offset + "0000000000000000000000000000000000000000000000000000000000000140" + // struct[d] offset + "0000000000000000000000000000000000000000000000000000000000000220" + // struct[e] offset + "0000000000000000000000000000000000000000000000000000000000000280" + // struct[f] offset + "0000000000000000000000000000000000000000000000000000000000000006" + // struct[a] length + "666f6f6261720000000000000000000000000000000000000000000000000000" + // struct[a] "foobar" + "0000000000000000000000000000000000000000000000000000000000000001" + // struct[c] length + "0100000000000000000000000000000000000000000000000000000000000000" + // []byte{1} + "0000000000000000000000000000000000000000000000000000000000000002" + // struct[d] length + "0000000000000000000000000000000000000000000000000000000000000040" + // foo offset + "0000000000000000000000000000000000000000000000000000000000000080" + // bar offset + "0000000000000000000000000000000000000000000000000000000000000003" + // foo length + "666f6f0000000000000000000000000000000000000000000000000000000000" + // foo + "0000000000000000000000000000000000000000000000000000000000000003" + // bar offset + "6261720000000000000000000000000000000000000000000000000000000000" + // bar + "0000000000000000000000000000000000000000000000000000000000000002" + // struct[e] length + "0000000000000000000000000000000000000000000000000000000000000001" + // 1 + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + // -1 + "0000000000000000000000000000000000000000000000000000000000000002" + // struct[f] length + "0000000000000000000000000100000000000000000000000000000000000000" + // common.Address{1} + "0000000000000000000000000200000000000000000000000000000000000000"), // common.Address{2} + }, + { + // nested tuple + "tuple", + []ArgumentMarshaling{ + {Name: "a", Type: "tuple", Components: []ArgumentMarshaling{{Name: "a", Type: "uint256"}, {Name: "b", Type: "uint256[]"}}}, + {Name: "b", Type: "int256[]"}, + }, + struct { + A struct { + FieldA *big.Int `abi:"a"` + B []*big.Int + } + B []*big.Int + }{ + A: struct { + FieldA *big.Int `abi:"a"` // Test whether abi tag works for nested tuple + B []*big.Int + }{big.NewInt(1), []*big.Int{big.NewInt(1), big.NewInt(0)}}, + B: []*big.Int{big.NewInt(1), big.NewInt(0)}}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // a offset + "00000000000000000000000000000000000000000000000000000000000000e0" + // b offset + "0000000000000000000000000000000000000000000000000000000000000001" + // a.a value + "0000000000000000000000000000000000000000000000000000000000000040" + // a.b offset + "0000000000000000000000000000000000000000000000000000000000000002" + // a.b length + "0000000000000000000000000000000000000000000000000000000000000001" + // a.b[0] value + "0000000000000000000000000000000000000000000000000000000000000000" + // a.b[1] value + "0000000000000000000000000000000000000000000000000000000000000002" + // b length + "0000000000000000000000000000000000000000000000000000000000000001" + // b[0] value + "0000000000000000000000000000000000000000000000000000000000000000"), // b[1] value + }, + { + // tuple slice + "tuple[]", + []ArgumentMarshaling{ + {Name: "a", Type: "int256"}, + {Name: "b", Type: "int256[]"}, + }, + []struct { + A *big.Int + B []*big.Int + }{ + {big.NewInt(-1), []*big.Int{big.NewInt(1), big.NewInt(0)}}, + {big.NewInt(1), []*big.Int{big.NewInt(2), big.NewInt(-1)}}, + }, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // tuple length + "0000000000000000000000000000000000000000000000000000000000000040" + // tuple[0] offset + "00000000000000000000000000000000000000000000000000000000000000e0" + // tuple[1] offset + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + // tuple[0].A + "0000000000000000000000000000000000000000000000000000000000000040" + // tuple[0].B offset + "0000000000000000000000000000000000000000000000000000000000000002" + // tuple[0].B length + "0000000000000000000000000000000000000000000000000000000000000001" + // tuple[0].B[0] value + "0000000000000000000000000000000000000000000000000000000000000000" + // tuple[0].B[1] value + "0000000000000000000000000000000000000000000000000000000000000001" + // tuple[1].A + "0000000000000000000000000000000000000000000000000000000000000040" + // tuple[1].B offset + "0000000000000000000000000000000000000000000000000000000000000002" + // tuple[1].B length + "0000000000000000000000000000000000000000000000000000000000000002" + // tuple[1].B[0] value + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), // tuple[1].B[1] value + }, + { + // static tuple array + "tuple[2]", + []ArgumentMarshaling{ + {Name: "a", Type: "int256"}, + {Name: "b", Type: "int256"}, + }, + [2]struct { + A *big.Int + B *big.Int + }{ + {big.NewInt(-1), big.NewInt(1)}, + {big.NewInt(1), big.NewInt(-1)}, + }, + common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + // tuple[0].a + "0000000000000000000000000000000000000000000000000000000000000001" + // tuple[0].b + "0000000000000000000000000000000000000000000000000000000000000001" + // tuple[1].a + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), // tuple[1].b + }, + { + // dynamic tuple array + "tuple[2]", + []ArgumentMarshaling{ + {Name: "a", Type: "int256[]"}, + }, + [2]struct { + A []*big.Int + }{ + {[]*big.Int{big.NewInt(-1), big.NewInt(1)}}, + {[]*big.Int{big.NewInt(1), big.NewInt(-1)}}, + }, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // tuple[0] offset + "00000000000000000000000000000000000000000000000000000000000000c0" + // tuple[1] offset + "0000000000000000000000000000000000000000000000000000000000000020" + // tuple[0].A offset + "0000000000000000000000000000000000000000000000000000000000000002" + // tuple[0].A length + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + // tuple[0].A[0] + "0000000000000000000000000000000000000000000000000000000000000001" + // tuple[0].A[1] + "0000000000000000000000000000000000000000000000000000000000000020" + // tuple[1].A offset + "0000000000000000000000000000000000000000000000000000000000000002" + // tuple[1].A length + "0000000000000000000000000000000000000000000000000000000000000001" + // tuple[1].A[0] + "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), // tuple[1].A[1] + }, } { - typ, err := NewType(test.typ) + typ, err := NewType(test.typ, test.components) if err != nil { t.Fatalf("%v failed. Unexpected parse error: %v", i, err) } - output, err := typ.pack(reflect.ValueOf(test.input)) if err != nil { t.Fatalf("%v failed. Unexpected pack error: %v", i, err) @@ -466,6 +693,59 @@ func TestMethodPack(t *testing.T) { if !bytes.Equal(packed, sig) { t.Errorf("expected %x got %x", sig, packed) } + + a := [2][2]*big.Int{{big.NewInt(1), big.NewInt(1)}, {big.NewInt(2), big.NewInt(0)}} + sig = abi.Methods["nestedArray"].Id() + sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{0}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{0xa0}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + sig = append(sig, common.LeftPadBytes(addrC[:], 32)...) + sig = append(sig, common.LeftPadBytes(addrD[:], 32)...) + packed, err = abi.Pack("nestedArray", a, []common.Address{addrC, addrD}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(packed, sig) { + t.Errorf("expected %x got %x", sig, packed) + } + + sig = abi.Methods["nestedArray2"].Id() + sig = append(sig, common.LeftPadBytes([]byte{0x20}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{0x40}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{0x80}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) + packed, err = abi.Pack("nestedArray2", [2][]uint8{{1}, {1}}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(packed, sig) { + t.Errorf("expected %x got %x", sig, packed) + } + + sig = abi.Methods["nestedSlice"].Id() + sig = append(sig, common.LeftPadBytes([]byte{0x20}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{0x02}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{0x40}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{0xa0}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{1}, 32)...) + sig = append(sig, common.LeftPadBytes([]byte{2}, 32)...) + packed, err = abi.Pack("nestedSlice", [][]uint8{{1, 2}, {1, 2}}) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(packed, sig) { + t.Errorf("expected %x got %x", sig, packed) + } } func TestPackNumber(t *testing.T) { diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index f541cf3bfe..1b0bb00496 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -71,18 +71,17 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value { // // set is a bit more lenient when it comes to assignment and doesn't force an as // strict ruleset as bare `reflect` does. -func set(dst, src reflect.Value, output Argument) error { - dstType := dst.Type() - srcType := src.Type() +func set(dst, src reflect.Value) error { + dstType, srcType := dst.Type(), src.Type() switch { - case dstType.AssignableTo(srcType): + case dstType.Kind() == reflect.Interface: + return set(dst.Elem(), src) + case dstType.Kind() == reflect.Ptr && dstType.Elem() != derefbigT: + return set(dst.Elem(), src) + case srcType.AssignableTo(dstType) && dst.CanSet(): dst.Set(src) case dstType.Kind() == reflect.Slice && srcType.Kind() == reflect.Slice: - return setSlice(dst, src, output) - case dstType.Kind() == reflect.Interface: - dst.Set(src) - case dstType.Kind() == reflect.Ptr: - return set(dst.Elem(), src, output) + return setSlice(dst, src) default: return fmt.Errorf("abi: cannot unmarshal %v in to %v", src.Type(), dst.Type()) } @@ -91,7 +90,7 @@ func set(dst, src reflect.Value, output Argument) error { // setSlice attempts to assign src to dst when slices are not assignable by default // e.g. src: [][]byte -> dst: [][15]byte -func setSlice(dst, src reflect.Value, output Argument) error { +func setSlice(dst, src reflect.Value) error { slice := reflect.MakeSlice(dst.Type(), src.Len(), src.Len()) for i := 0; i < src.Len(); i++ { v := src.Index(i) @@ -127,14 +126,14 @@ func requireUnpackKind(v reflect.Value, t reflect.Type, k reflect.Kind, return nil } -// mapAbiToStringField maps abi to struct fields. +// mapArgNamesToStructFields maps a slice of argument names to struct fields. // first round: for each Exportable field that contains a `abi:""` tag -// and this field name exists in the arguments, pair them together. -// second round: for each argument field that has not been already linked, +// and this field name exists in the given argument name list, pair them together. +// second round: for each argument name that has not been already linked, // find what variable is expected to be mapped into, if it exists and has not been // used, pair them. -func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]string, error) { - +// Note this function assumes the given value is a struct value. +func mapArgNamesToStructFields(argNames []string, value reflect.Value) (map[string]string, error) { typ := value.Type() abi2struct := make(map[string]string) @@ -148,45 +147,39 @@ func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]strin if structFieldName[:1] != strings.ToUpper(structFieldName[:1]) { continue } - // skip fields that have no abi:"" tag. var ok bool var tagName string if tagName, ok = typ.Field(i).Tag.Lookup("abi"); !ok { continue } - // check if tag is empty. if tagName == "" { return nil, fmt.Errorf("struct: abi tag in '%s' is empty", structFieldName) } - // check which argument field matches with the abi tag. found := false - for _, abiField := range args.NonIndexed() { - if abiField.Name == tagName { - if abi2struct[abiField.Name] != "" { + for _, arg := range argNames { + if arg == tagName { + if abi2struct[arg] != "" { return nil, fmt.Errorf("struct: abi tag in '%s' already mapped", structFieldName) } // pair them - abi2struct[abiField.Name] = structFieldName - struct2abi[structFieldName] = abiField.Name + abi2struct[arg] = structFieldName + struct2abi[structFieldName] = arg found = true } } - // check if this tag has been mapped. if !found { return nil, fmt.Errorf("struct: abi tag '%s' defined but not found in abi", tagName) } - } // second round ~~~ - for _, arg := range args { + for _, argName := range argNames { - abiFieldName := arg.Name - structFieldName := ToCamelCase(abiFieldName) + structFieldName := ToCamelCase(argName) if structFieldName == "" { return nil, fmt.Errorf("abi: purely underscored output cannot unpack to struct") @@ -196,11 +189,11 @@ func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]strin // struct field with the same field name. If so, raise an error: // abi: [ { "name": "value" } ] // struct { Value *big.Int , Value1 *big.Int `abi:"value"`} - if abi2struct[abiFieldName] != "" { - if abi2struct[abiFieldName] != structFieldName && + if abi2struct[argName] != "" { + if abi2struct[argName] != structFieldName && struct2abi[structFieldName] == "" && value.FieldByName(structFieldName).IsValid() { - return nil, fmt.Errorf("abi: multiple variables maps to the same abi field '%s'", abiFieldName) + return nil, fmt.Errorf("abi: multiple variables maps to the same abi field '%s'", argName) } continue } @@ -212,16 +205,14 @@ func mapAbiToStructFields(args Arguments, value reflect.Value) (map[string]strin if value.FieldByName(structFieldName).IsValid() { // pair them - abi2struct[abiFieldName] = structFieldName - struct2abi[structFieldName] = abiFieldName + abi2struct[argName] = structFieldName + struct2abi[structFieldName] = argName } else { // not paired, but annotate as used, to detect cases like // abi : [ { "name": "value" }, { "name": "_value" } ] // struct { Value *big.Int } - struct2abi[structFieldName] = abiFieldName + struct2abi[structFieldName] = argName } - } - return abi2struct, nil } diff --git a/accounts/abi/type.go b/accounts/abi/type.go index 6bfaabf5a5..26151dbd3e 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -17,6 +17,7 @@ package abi import ( + "errors" "fmt" "reflect" "regexp" @@ -32,6 +33,7 @@ const ( StringTy SliceTy ArrayTy + TupleTy AddressTy FixedBytesTy BytesTy @@ -43,13 +45,16 @@ const ( // Type is the reflection of the supported argument type type Type struct { Elem *Type - Kind reflect.Kind Type reflect.Type Size int T byte // Our own type checking stringKind string // holds the unparsed string for deriving signatures + + // Tuple relative fields + TupleElems []*Type // Type information of all tuple fields + TupleRawNames []string // Raw field name of all tuple fields } var ( @@ -58,7 +63,7 @@ var ( ) // NewType creates a new reflection type of abi type given in t. -func NewType(t string) (typ Type, err error) { +func NewType(t string, components []ArgumentMarshaling) (typ Type, err error) { // check that array brackets are equal if they exist if strings.Count(t, "[") != strings.Count(t, "]") { return Type{}, fmt.Errorf("invalid arg type in abi") @@ -71,7 +76,7 @@ func NewType(t string) (typ Type, err error) { if strings.Count(t, "[") != 0 { i := strings.LastIndex(t, "[") // recursively embed the type - embeddedType, err := NewType(t[:i]) + embeddedType, err := NewType(t[:i], components) if err != nil { return Type{}, err } @@ -87,6 +92,9 @@ func NewType(t string) (typ Type, err error) { typ.Kind = reflect.Slice typ.Elem = &embeddedType typ.Type = reflect.SliceOf(embeddedType.Type) + if embeddedType.T == TupleTy { + typ.stringKind = embeddedType.stringKind + sliced + } } else if len(intz) == 1 { // is a array typ.T = ArrayTy @@ -97,6 +105,9 @@ func NewType(t string) (typ Type, err error) { return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err) } typ.Type = reflect.ArrayOf(typ.Size, embeddedType.Type) + if embeddedType.T == TupleTy { + typ.stringKind = embeddedType.stringKind + sliced + } } else { return Type{}, fmt.Errorf("invalid formatting of array type") } @@ -158,6 +169,40 @@ func NewType(t string) (typ Type, err error) { typ.Size = varSize typ.Type = reflect.ArrayOf(varSize, reflect.TypeOf(byte(0))) } + case "tuple": + var ( + fields []reflect.StructField + elems []*Type + names []string + expression string // canonical parameter expression + ) + expression += "(" + for idx, c := range components { + cType, err := NewType(c.Type, c.Components) + if err != nil { + return Type{}, err + } + if ToCamelCase(c.Name) == "" { + return Type{}, errors.New("abi: purely anonymous or underscored field is not supported") + } + fields = append(fields, reflect.StructField{ + Name: ToCamelCase(c.Name), // reflect.StructOf will panic for any exported field. + Type: cType.Type, + }) + elems = append(elems, &cType) + names = append(names, c.Name) + expression += cType.stringKind + if idx != len(components)-1 { + expression += "," + } + } + expression += ")" + typ.Kind = reflect.Struct + typ.Type = reflect.StructOf(fields) + typ.TupleElems = elems + typ.TupleRawNames = names + typ.T = TupleTy + typ.stringKind = expression case "function": typ.Kind = reflect.Array typ.T = FunctionTy @@ -178,7 +223,6 @@ func (t Type) String() (out string) { func (t Type) pack(v reflect.Value) ([]byte, error) { // dereference pointer first if it's a pointer v = indirect(v) - if err := typeCheck(t, v); err != nil { return nil, err } @@ -196,7 +240,7 @@ func (t Type) pack(v reflect.Value) ([]byte, error) { offset := 0 offsetReq := isDynamicType(*t.Elem) if offsetReq { - offset = getDynamicTypeOffset(*t.Elem) * v.Len() + offset = getTypeSize(*t.Elem) * v.Len() } var tail []byte for i := 0; i < v.Len(); i++ { @@ -213,6 +257,45 @@ func (t Type) pack(v reflect.Value) ([]byte, error) { tail = append(tail, val...) } return append(ret, tail...), nil + case TupleTy: + // (T1,...,Tk) for k >= 0 and any types T1, …, Tk + // enc(X) = head(X(1)) ... head(X(k)) tail(X(1)) ... tail(X(k)) + // where X = (X(1), ..., X(k)) and head and tail are defined for Ti being a static + // type as + // head(X(i)) = enc(X(i)) and tail(X(i)) = "" (the empty string) + // and as + // head(X(i)) = enc(len(head(X(1)) ... head(X(k)) tail(X(1)) ... tail(X(i-1)))) + // tail(X(i)) = enc(X(i)) + // otherwise, i.e. if Ti is a dynamic type. + fieldmap, err := mapArgNamesToStructFields(t.TupleRawNames, v) + if err != nil { + return nil, err + } + // Calculate prefix occupied size. + offset := 0 + for _, elem := range t.TupleElems { + offset += getTypeSize(*elem) + } + var ret, tail []byte + for i, elem := range t.TupleElems { + field := v.FieldByName(fieldmap[t.TupleRawNames[i]]) + if !field.IsValid() { + return nil, fmt.Errorf("field %s for tuple not found in the given struct", t.TupleRawNames[i]) + } + val, err := elem.pack(field) + if err != nil { + return nil, err + } + if isDynamicType(*elem) { + ret = append(ret, packNum(reflect.ValueOf(offset))...) + tail = append(tail, val...) + offset += len(val) + } else { + ret = append(ret, val...) + } + } + return append(ret, tail...), nil + default: return packElement(t, v), nil } @@ -225,25 +308,45 @@ func (t Type) requiresLengthPrefix() bool { } // isDynamicType returns true if the type is dynamic. -// StringTy, BytesTy, and SliceTy(irrespective of slice element type) are dynamic types -// ArrayTy is considered dynamic if and only if the Array element is a dynamic type. -// This function recursively checks the type for slice and array elements. +// The following types are called “dynamic”: +// * bytes +// * string +// * T[] for any T +// * T[k] for any dynamic T and any k >= 0 +// * (T1,...,Tk) if Ti is dynamic for some 1 <= i <= k func isDynamicType(t Type) bool { - // dynamic types - // array is also a dynamic type if the array type is dynamic + if t.T == TupleTy { + for _, elem := range t.TupleElems { + if isDynamicType(*elem) { + return true + } + } + return false + } return t.T == StringTy || t.T == BytesTy || t.T == SliceTy || (t.T == ArrayTy && isDynamicType(*t.Elem)) } -// getDynamicTypeOffset returns the offset for the type. -// See `isDynamicType` to know which types are considered dynamic. -// If the type t is an array and element type is not a dynamic type, then we consider it a static type and -// return 32 * size of array since length prefix is not required. -// If t is a dynamic type or element type(for slices and arrays) is dynamic, then we simply return 32 as offset. -func getDynamicTypeOffset(t Type) int { - // if it is an array and there are no dynamic types - // then the array is static type +// getTypeSize returns the size that this type needs to occupy. +// We distinguish static and dynamic types. Static types are encoded in-place +// and dynamic types are encoded at a separately allocated location after the +// current block. +// So for a static variable, the size returned represents the size that the +// variable actually occupies. +// For a dynamic variable, the returned size is fixed 32 bytes, which is used +// to store the location reference for actual value storage. +func getTypeSize(t Type) int { if t.T == ArrayTy && !isDynamicType(*t.Elem) { - return 32 * t.Size + // Recursively calculate type size if it is a nested array + if t.Elem.T == ArrayTy { + return t.Size * getTypeSize(*t.Elem) + } + return t.Size * 32 + } else if t.T == TupleTy && !isDynamicType(t) { + total := 0 + for _, elem := range t.TupleElems { + total += getTypeSize(*elem) + } + return total } return 32 } diff --git a/accounts/abi/type_test.go b/accounts/abi/type_test.go index f6b36f18fd..7ef47330dc 100644 --- a/accounts/abi/type_test.go +++ b/accounts/abi/type_test.go @@ -32,72 +32,75 @@ type typeWithoutStringer Type // Tests that all allowed types get recognized by the type parser. func TestTypeRegexp(t *testing.T) { tests := []struct { - blob string - kind Type + blob string + components []ArgumentMarshaling + kind Type }{ - {"bool", Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}}, - {"bool[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool(nil)), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}}, - {"bool[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}}, - {"bool[2][]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}}, - {"bool[][]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}}, - {"bool[][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}}, - {"bool[2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}}, - {"bool[2][][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][][2]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}, stringKind: "bool[2][][2]"}}, - {"bool[2][2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}}, - {"bool[][][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}}, - {"bool[][2][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}}, - {"int8", Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}}, - {"int16", Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}}, - {"int32", Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}}, - {"int64", Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}}, - {"int256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}}, - {"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}}, - {"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}}, - {"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}}, - {"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}}, - {"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}}, - {"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}}, - {"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}}, - {"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}}, - {"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}}, - {"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}}, - {"uint8", Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}}, - {"uint16", Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}}, - {"uint32", Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}}, - {"uint64", Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}}, - {"uint256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}}, - {"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}}, - {"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}}, - {"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}}, - {"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}}, - {"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}}, - {"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}}, - {"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}}, - {"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}}, - {"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}}, - {"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}}, - {"bytes32", Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}}, - {"bytes[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}}, - {"bytes[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}}, - {"bytes32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][32]byte{}), Elem: &Type{Kind: reflect.Array, Type: reflect.TypeOf([32]byte{}), T: FixedBytesTy, Size: 32, stringKind: "bytes32"}, stringKind: "bytes32[]"}}, - {"bytes32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][32]byte{}), Elem: &Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}, stringKind: "bytes32[2]"}}, - {"string", Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}}, - {"string[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}}, - {"string[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}}, - {"address", Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}}, - {"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}}, - {"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}}, + {"bool", nil, Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}}, + {"bool[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool(nil)), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}}, + {"bool[2]", nil, Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}}, + {"bool[2][]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}}, + {"bool[][]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}}, + {"bool[][2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}}, + {"bool[2][2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}}, + {"bool[2][][2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][][2]bool{}), Elem: &Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][]"}, stringKind: "bool[2][][2]"}}, + {"bool[2][2][2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}}, + {"bool[][][]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}}, + {"bool[][2][]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}}, + {"int8", nil, Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}}, + {"int16", nil, Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}}, + {"int32", nil, Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}}, + {"int64", nil, Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}}, + {"int256", nil, Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}}, + {"int8[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}}, + {"int8[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}}, + {"int16[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}}, + {"int16[2]", nil, Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}}, + {"int32[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}}, + {"int32[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}}, + {"int64[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}}, + {"int64[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}}, + {"int256[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}}, + {"int256[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}}, + {"uint8", nil, Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}}, + {"uint16", nil, Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}}, + {"uint32", nil, Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}}, + {"uint64", nil, Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}}, + {"uint256", nil, Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}}, + {"uint8[]", nil, Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}}, + {"uint8[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}}, + {"uint16[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}}, + {"uint16[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}}, + {"uint32[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}}, + {"uint32[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}}, + {"uint64[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}}, + {"uint64[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}}, + {"uint256[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}}, + {"uint256[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}}, + {"bytes32", nil, Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}}, + {"bytes[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}}, + {"bytes[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}}, + {"bytes32[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][32]byte{}), Elem: &Type{Kind: reflect.Array, Type: reflect.TypeOf([32]byte{}), T: FixedBytesTy, Size: 32, stringKind: "bytes32"}, stringKind: "bytes32[]"}}, + {"bytes32[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][32]byte{}), Elem: &Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}, stringKind: "bytes32[2]"}}, + {"string", nil, Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}}, + {"string[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}}, + {"string[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}}, + {"address", nil, Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}}, + {"address[]", nil, Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}}, + {"address[2]", nil, Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}}, // TODO when fixed types are implemented properly - // {"fixed", Type{}}, - // {"fixed128x128", Type{}}, - // {"fixed[]", Type{}}, - // {"fixed[2]", Type{}}, - // {"fixed128x128[]", Type{}}, - // {"fixed128x128[2]", Type{}}, + // {"fixed", nil, Type{}}, + // {"fixed128x128", nil, Type{}}, + // {"fixed[]", nil, Type{}}, + // {"fixed[2]", nil, Type{}}, + // {"fixed128x128[]", nil, Type{}}, + // {"fixed128x128[2]", nil, Type{}}, + {"tuple", []ArgumentMarshaling{{Name: "a", Type: "int64"}}, Type{Kind: reflect.Struct, T: TupleTy, Type: reflect.TypeOf(struct{ A int64 }{}), stringKind: "(int64)", + TupleElems: []*Type{{Kind: reflect.Int64, T: IntTy, Type: reflect.TypeOf(int64(0)), Size: 64, stringKind: "int64"}}, TupleRawNames: []string{"a"}}}, } for _, tt := range tests { - typ, err := NewType(tt.blob) + typ, err := NewType(tt.blob, tt.components) if err != nil { t.Errorf("type %q: failed to parse type string: %v", tt.blob, err) } @@ -109,154 +112,170 @@ func TestTypeRegexp(t *testing.T) { func TestTypeCheck(t *testing.T) { for i, test := range []struct { - typ string - input interface{} - err string + typ string + components []ArgumentMarshaling + input interface{} + err string }{ - {"uint", big.NewInt(1), "unsupported arg type: uint"}, - {"int", big.NewInt(1), "unsupported arg type: int"}, - {"uint256", big.NewInt(1), ""}, - {"uint256[][3][]", [][3][]*big.Int{{{}}}, ""}, - {"uint256[][][3]", [3][][]*big.Int{{{}}}, ""}, - {"uint256[3][][]", [][][3]*big.Int{{{}}}, ""}, - {"uint256[3][3][3]", [3][3][3]*big.Int{{{}}}, ""}, - {"uint8[][]", [][]uint8{}, ""}, - {"int256", big.NewInt(1), ""}, - {"uint8", uint8(1), ""}, - {"uint16", uint16(1), ""}, - {"uint32", uint32(1), ""}, - {"uint64", uint64(1), ""}, - {"int8", int8(1), ""}, - {"int16", int16(1), ""}, - {"int32", int32(1), ""}, - {"int64", int64(1), ""}, - {"uint24", big.NewInt(1), ""}, - {"uint40", big.NewInt(1), ""}, - {"uint48", big.NewInt(1), ""}, - {"uint56", big.NewInt(1), ""}, - {"uint72", big.NewInt(1), ""}, - {"uint80", big.NewInt(1), ""}, - {"uint88", big.NewInt(1), ""}, - {"uint96", big.NewInt(1), ""}, - {"uint104", big.NewInt(1), ""}, - {"uint112", big.NewInt(1), ""}, - {"uint120", big.NewInt(1), ""}, - {"uint128", big.NewInt(1), ""}, - {"uint136", big.NewInt(1), ""}, - {"uint144", big.NewInt(1), ""}, - {"uint152", big.NewInt(1), ""}, - {"uint160", big.NewInt(1), ""}, - {"uint168", big.NewInt(1), ""}, - {"uint176", big.NewInt(1), ""}, - {"uint184", big.NewInt(1), ""}, - {"uint192", big.NewInt(1), ""}, - {"uint200", big.NewInt(1), ""}, - {"uint208", big.NewInt(1), ""}, - {"uint216", big.NewInt(1), ""}, - {"uint224", big.NewInt(1), ""}, - {"uint232", big.NewInt(1), ""}, - {"uint240", big.NewInt(1), ""}, - {"uint248", big.NewInt(1), ""}, - {"int24", big.NewInt(1), ""}, - {"int40", big.NewInt(1), ""}, - {"int48", big.NewInt(1), ""}, - {"int56", big.NewInt(1), ""}, - {"int72", big.NewInt(1), ""}, - {"int80", big.NewInt(1), ""}, - {"int88", big.NewInt(1), ""}, - {"int96", big.NewInt(1), ""}, - {"int104", big.NewInt(1), ""}, - {"int112", big.NewInt(1), ""}, - {"int120", big.NewInt(1), ""}, - {"int128", big.NewInt(1), ""}, - {"int136", big.NewInt(1), ""}, - {"int144", big.NewInt(1), ""}, - {"int152", big.NewInt(1), ""}, - {"int160", big.NewInt(1), ""}, - {"int168", big.NewInt(1), ""}, - {"int176", big.NewInt(1), ""}, - {"int184", big.NewInt(1), ""}, - {"int192", big.NewInt(1), ""}, - {"int200", big.NewInt(1), ""}, - {"int208", big.NewInt(1), ""}, - {"int216", big.NewInt(1), ""}, - {"int224", big.NewInt(1), ""}, - {"int232", big.NewInt(1), ""}, - {"int240", big.NewInt(1), ""}, - {"int248", big.NewInt(1), ""}, - {"uint30", uint8(1), "abi: cannot use uint8 as type ptr as argument"}, - {"uint8", uint16(1), "abi: cannot use uint16 as type uint8 as argument"}, - {"uint8", uint32(1), "abi: cannot use uint32 as type uint8 as argument"}, - {"uint8", uint64(1), "abi: cannot use uint64 as type uint8 as argument"}, - {"uint8", int8(1), "abi: cannot use int8 as type uint8 as argument"}, - {"uint8", int16(1), "abi: cannot use int16 as type uint8 as argument"}, - {"uint8", int32(1), "abi: cannot use int32 as type uint8 as argument"}, - {"uint8", int64(1), "abi: cannot use int64 as type uint8 as argument"}, - {"uint16", uint16(1), ""}, - {"uint16", uint8(1), "abi: cannot use uint8 as type uint16 as argument"}, - {"uint16[]", []uint16{1, 2, 3}, ""}, - {"uint16[]", [3]uint16{1, 2, 3}, ""}, - {"uint16[]", []uint32{1, 2, 3}, "abi: cannot use []uint32 as type [0]uint16 as argument"}, - {"uint16[3]", [3]uint32{1, 2, 3}, "abi: cannot use [3]uint32 as type [3]uint16 as argument"}, - {"uint16[3]", [4]uint16{1, 2, 3}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"}, - {"uint16[3]", []uint16{1, 2, 3}, ""}, - {"uint16[3]", []uint16{1, 2, 3, 4}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"}, - {"address[]", []common.Address{{1}}, ""}, - {"address[1]", []common.Address{{1}}, ""}, - {"address[1]", [1]common.Address{{1}}, ""}, - {"address[2]", [1]common.Address{{1}}, "abi: cannot use [1]array as type [2]array as argument"}, - {"bytes32", [32]byte{}, ""}, - {"bytes31", [31]byte{}, ""}, - {"bytes30", [30]byte{}, ""}, - {"bytes29", [29]byte{}, ""}, - {"bytes28", [28]byte{}, ""}, - {"bytes27", [27]byte{}, ""}, - {"bytes26", [26]byte{}, ""}, - {"bytes25", [25]byte{}, ""}, - {"bytes24", [24]byte{}, ""}, - {"bytes23", [23]byte{}, ""}, - {"bytes22", [22]byte{}, ""}, - {"bytes21", [21]byte{}, ""}, - {"bytes20", [20]byte{}, ""}, - {"bytes19", [19]byte{}, ""}, - {"bytes18", [18]byte{}, ""}, - {"bytes17", [17]byte{}, ""}, - {"bytes16", [16]byte{}, ""}, - {"bytes15", [15]byte{}, ""}, - {"bytes14", [14]byte{}, ""}, - {"bytes13", [13]byte{}, ""}, - {"bytes12", [12]byte{}, ""}, - {"bytes11", [11]byte{}, ""}, - {"bytes10", [10]byte{}, ""}, - {"bytes9", [9]byte{}, ""}, - {"bytes8", [8]byte{}, ""}, - {"bytes7", [7]byte{}, ""}, - {"bytes6", [6]byte{}, ""}, - {"bytes5", [5]byte{}, ""}, - {"bytes4", [4]byte{}, ""}, - {"bytes3", [3]byte{}, ""}, - {"bytes2", [2]byte{}, ""}, - {"bytes1", [1]byte{}, ""}, - {"bytes32", [33]byte{}, "abi: cannot use [33]uint8 as type [32]uint8 as argument"}, - {"bytes32", common.Hash{1}, ""}, - {"bytes31", common.Hash{1}, "abi: cannot use common.Hash as type [31]uint8 as argument"}, - {"bytes31", [32]byte{}, "abi: cannot use [32]uint8 as type [31]uint8 as argument"}, - {"bytes", []byte{0, 1}, ""}, - {"bytes", [2]byte{0, 1}, "abi: cannot use array as type slice as argument"}, - {"bytes", common.Hash{1}, "abi: cannot use array as type slice as argument"}, - {"string", "hello world", ""}, - {"string", string(""), ""}, - {"string", []byte{}, "abi: cannot use slice as type string as argument"}, - {"bytes32[]", [][32]byte{{}}, ""}, - {"function", [24]byte{}, ""}, - {"bytes20", common.Address{}, ""}, - {"address", [20]byte{}, ""}, - {"address", common.Address{}, ""}, - {"bytes32[]]", "", "invalid arg type in abi"}, - {"invalidType", "", "unsupported arg type: invalidType"}, - {"invalidSlice[]", "", "unsupported arg type: invalidSlice"}, + {"uint", nil, big.NewInt(1), "unsupported arg type: uint"}, + {"int", nil, big.NewInt(1), "unsupported arg type: int"}, + {"uint256", nil, big.NewInt(1), ""}, + {"uint256[][3][]", nil, [][3][]*big.Int{{{}}}, ""}, + {"uint256[][][3]", nil, [3][][]*big.Int{{{}}}, ""}, + {"uint256[3][][]", nil, [][][3]*big.Int{{{}}}, ""}, + {"uint256[3][3][3]", nil, [3][3][3]*big.Int{{{}}}, ""}, + {"uint8[][]", nil, [][]uint8{}, ""}, + {"int256", nil, big.NewInt(1), ""}, + {"uint8", nil, uint8(1), ""}, + {"uint16", nil, uint16(1), ""}, + {"uint32", nil, uint32(1), ""}, + {"uint64", nil, uint64(1), ""}, + {"int8", nil, int8(1), ""}, + {"int16", nil, int16(1), ""}, + {"int32", nil, int32(1), ""}, + {"int64", nil, int64(1), ""}, + {"uint24", nil, big.NewInt(1), ""}, + {"uint40", nil, big.NewInt(1), ""}, + {"uint48", nil, big.NewInt(1), ""}, + {"uint56", nil, big.NewInt(1), ""}, + {"uint72", nil, big.NewInt(1), ""}, + {"uint80", nil, big.NewInt(1), ""}, + {"uint88", nil, big.NewInt(1), ""}, + {"uint96", nil, big.NewInt(1), ""}, + {"uint104", nil, big.NewInt(1), ""}, + {"uint112", nil, big.NewInt(1), ""}, + {"uint120", nil, big.NewInt(1), ""}, + {"uint128", nil, big.NewInt(1), ""}, + {"uint136", nil, big.NewInt(1), ""}, + {"uint144", nil, big.NewInt(1), ""}, + {"uint152", nil, big.NewInt(1), ""}, + {"uint160", nil, big.NewInt(1), ""}, + {"uint168", nil, big.NewInt(1), ""}, + {"uint176", nil, big.NewInt(1), ""}, + {"uint184", nil, big.NewInt(1), ""}, + {"uint192", nil, big.NewInt(1), ""}, + {"uint200", nil, big.NewInt(1), ""}, + {"uint208", nil, big.NewInt(1), ""}, + {"uint216", nil, big.NewInt(1), ""}, + {"uint224", nil, big.NewInt(1), ""}, + {"uint232", nil, big.NewInt(1), ""}, + {"uint240", nil, big.NewInt(1), ""}, + {"uint248", nil, big.NewInt(1), ""}, + {"int24", nil, big.NewInt(1), ""}, + {"int40", nil, big.NewInt(1), ""}, + {"int48", nil, big.NewInt(1), ""}, + {"int56", nil, big.NewInt(1), ""}, + {"int72", nil, big.NewInt(1), ""}, + {"int80", nil, big.NewInt(1), ""}, + {"int88", nil, big.NewInt(1), ""}, + {"int96", nil, big.NewInt(1), ""}, + {"int104", nil, big.NewInt(1), ""}, + {"int112", nil, big.NewInt(1), ""}, + {"int120", nil, big.NewInt(1), ""}, + {"int128", nil, big.NewInt(1), ""}, + {"int136", nil, big.NewInt(1), ""}, + {"int144", nil, big.NewInt(1), ""}, + {"int152", nil, big.NewInt(1), ""}, + {"int160", nil, big.NewInt(1), ""}, + {"int168", nil, big.NewInt(1), ""}, + {"int176", nil, big.NewInt(1), ""}, + {"int184", nil, big.NewInt(1), ""}, + {"int192", nil, big.NewInt(1), ""}, + {"int200", nil, big.NewInt(1), ""}, + {"int208", nil, big.NewInt(1), ""}, + {"int216", nil, big.NewInt(1), ""}, + {"int224", nil, big.NewInt(1), ""}, + {"int232", nil, big.NewInt(1), ""}, + {"int240", nil, big.NewInt(1), ""}, + {"int248", nil, big.NewInt(1), ""}, + {"uint30", nil, uint8(1), "abi: cannot use uint8 as type ptr as argument"}, + {"uint8", nil, uint16(1), "abi: cannot use uint16 as type uint8 as argument"}, + {"uint8", nil, uint32(1), "abi: cannot use uint32 as type uint8 as argument"}, + {"uint8", nil, uint64(1), "abi: cannot use uint64 as type uint8 as argument"}, + {"uint8", nil, int8(1), "abi: cannot use int8 as type uint8 as argument"}, + {"uint8", nil, int16(1), "abi: cannot use int16 as type uint8 as argument"}, + {"uint8", nil, int32(1), "abi: cannot use int32 as type uint8 as argument"}, + {"uint8", nil, int64(1), "abi: cannot use int64 as type uint8 as argument"}, + {"uint16", nil, uint16(1), ""}, + {"uint16", nil, uint8(1), "abi: cannot use uint8 as type uint16 as argument"}, + {"uint16[]", nil, []uint16{1, 2, 3}, ""}, + {"uint16[]", nil, [3]uint16{1, 2, 3}, ""}, + {"uint16[]", nil, []uint32{1, 2, 3}, "abi: cannot use []uint32 as type [0]uint16 as argument"}, + {"uint16[3]", nil, [3]uint32{1, 2, 3}, "abi: cannot use [3]uint32 as type [3]uint16 as argument"}, + {"uint16[3]", nil, [4]uint16{1, 2, 3}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"}, + {"uint16[3]", nil, []uint16{1, 2, 3}, ""}, + {"uint16[3]", nil, []uint16{1, 2, 3, 4}, "abi: cannot use [4]uint16 as type [3]uint16 as argument"}, + {"address[]", nil, []common.Address{{1}}, ""}, + {"address[1]", nil, []common.Address{{1}}, ""}, + {"address[1]", nil, [1]common.Address{{1}}, ""}, + {"address[2]", nil, [1]common.Address{{1}}, "abi: cannot use [1]array as type [2]array as argument"}, + {"bytes32", nil, [32]byte{}, ""}, + {"bytes31", nil, [31]byte{}, ""}, + {"bytes30", nil, [30]byte{}, ""}, + {"bytes29", nil, [29]byte{}, ""}, + {"bytes28", nil, [28]byte{}, ""}, + {"bytes27", nil, [27]byte{}, ""}, + {"bytes26", nil, [26]byte{}, ""}, + {"bytes25", nil, [25]byte{}, ""}, + {"bytes24", nil, [24]byte{}, ""}, + {"bytes23", nil, [23]byte{}, ""}, + {"bytes22", nil, [22]byte{}, ""}, + {"bytes21", nil, [21]byte{}, ""}, + {"bytes20", nil, [20]byte{}, ""}, + {"bytes19", nil, [19]byte{}, ""}, + {"bytes18", nil, [18]byte{}, ""}, + {"bytes17", nil, [17]byte{}, ""}, + {"bytes16", nil, [16]byte{}, ""}, + {"bytes15", nil, [15]byte{}, ""}, + {"bytes14", nil, [14]byte{}, ""}, + {"bytes13", nil, [13]byte{}, ""}, + {"bytes12", nil, [12]byte{}, ""}, + {"bytes11", nil, [11]byte{}, ""}, + {"bytes10", nil, [10]byte{}, ""}, + {"bytes9", nil, [9]byte{}, ""}, + {"bytes8", nil, [8]byte{}, ""}, + {"bytes7", nil, [7]byte{}, ""}, + {"bytes6", nil, [6]byte{}, ""}, + {"bytes5", nil, [5]byte{}, ""}, + {"bytes4", nil, [4]byte{}, ""}, + {"bytes3", nil, [3]byte{}, ""}, + {"bytes2", nil, [2]byte{}, ""}, + {"bytes1", nil, [1]byte{}, ""}, + {"bytes32", nil, [33]byte{}, "abi: cannot use [33]uint8 as type [32]uint8 as argument"}, + {"bytes32", nil, common.Hash{1}, ""}, + {"bytes31", nil, common.Hash{1}, "abi: cannot use common.Hash as type [31]uint8 as argument"}, + {"bytes31", nil, [32]byte{}, "abi: cannot use [32]uint8 as type [31]uint8 as argument"}, + {"bytes", nil, []byte{0, 1}, ""}, + {"bytes", nil, [2]byte{0, 1}, "abi: cannot use array as type slice as argument"}, + {"bytes", nil, common.Hash{1}, "abi: cannot use array as type slice as argument"}, + {"string", nil, "hello world", ""}, + {"string", nil, string(""), ""}, + {"string", nil, []byte{}, "abi: cannot use slice as type string as argument"}, + {"bytes32[]", nil, [][32]byte{{}}, ""}, + {"function", nil, [24]byte{}, ""}, + {"bytes20", nil, common.Address{}, ""}, + {"address", nil, [20]byte{}, ""}, + {"address", nil, common.Address{}, ""}, + {"bytes32[]]", nil, "", "invalid arg type in abi"}, + {"invalidType", nil, "", "unsupported arg type: invalidType"}, + {"invalidSlice[]", nil, "", "unsupported arg type: invalidSlice"}, + // simple tuple + {"tuple", []ArgumentMarshaling{{Name: "a", Type: "uint256"}, {Name: "b", Type: "uint256"}}, struct { + A *big.Int + B *big.Int + }{}, ""}, + // tuple slice + {"tuple[]", []ArgumentMarshaling{{Name: "a", Type: "uint256"}, {Name: "b", Type: "uint256"}}, []struct { + A *big.Int + B *big.Int + }{}, ""}, + // tuple array + {"tuple[2]", []ArgumentMarshaling{{Name: "a", Type: "uint256"}, {Name: "b", Type: "uint256"}}, []struct { + A *big.Int + B *big.Int + }{{big.NewInt(0), big.NewInt(0)}, {big.NewInt(0), big.NewInt(0)}}, ""}, } { - typ, err := NewType(test.typ) + typ, err := NewType(test.typ, test.components) if err != nil && len(test.err) == 0 { t.Fatal("unexpected parse error:", err) } else if err != nil && len(test.err) != 0 { diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 04716f7a2a..8406b09c80 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -115,17 +115,6 @@ func readFixedBytes(t Type, word []byte) (interface{}, error) { } -func getFullElemSize(elem *Type) int { - //all other should be counted as 32 (slices have pointers to respective elements) - size := 32 - //arrays wrap it, each element being the same size - for elem.T == ArrayTy { - size *= elem.Size - elem = elem.Elem - } - return size -} - // iteratively unpack elements func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) { if size < 0 { @@ -150,13 +139,9 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) // Arrays have packed elements, resulting in longer unpack steps. // Slices have just 32 bytes per element (pointing to the contents). - elemSize := 32 - if t.T == ArrayTy || t.T == SliceTy { - elemSize = getFullElemSize(t.Elem) - } + elemSize := getTypeSize(*t.Elem) for i, j := start, 0; j < size; i, j = i+elemSize, j+1 { - inter, err := toGoType(i, *t.Elem, output) if err != nil { return nil, err @@ -170,6 +155,36 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) return refSlice.Interface(), nil } +func forTupleUnpack(t Type, output []byte) (interface{}, error) { + retval := reflect.New(t.Type).Elem() + virtualArgs := 0 + for index, elem := range t.TupleElems { + marshalledValue, err := toGoType((index+virtualArgs)*32, *elem, output) + if elem.T == ArrayTy && !isDynamicType(*elem) { + // If we have a static array, like [3]uint256, these are coded as + // just like uint256,uint256,uint256. + // This means that we need to add two 'virtual' arguments when + // we count the index from now on. + // + // Array values nested multiple levels deep are also encoded inline: + // [2][3]uint256: uint256,uint256,uint256,uint256,uint256,uint256 + // + // Calculate the full array size to get the correct offset for the next argument. + // Decrement it by 1, as the normal index increment is still applied. + virtualArgs += getTypeSize(*elem)/32 - 1 + } else if elem.T == TupleTy && !isDynamicType(*elem) { + // If we have a static tuple, like (uint256, bool, uint256), these are + // coded as just like uint256,bool,uint256 + virtualArgs += getTypeSize(*elem)/32 - 1 + } + if err != nil { + return nil, err + } + retval.Field(index).Set(reflect.ValueOf(marshalledValue)) + } + return retval.Interface(), nil +} + // toGoType parses the output bytes and recursively assigns the value of these bytes // into a go type with accordance with the ABI spec. func toGoType(index int, t Type, output []byte) (interface{}, error) { @@ -178,14 +193,14 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { } var ( - returnOutput []byte - begin, end int - err error + returnOutput []byte + begin, length int + err error ) // if we require a length prefix, find the beginning word and size returned. if t.requiresLengthPrefix() { - begin, end, err = lengthPrefixPointsTo(index, output) + begin, length, err = lengthPrefixPointsTo(index, output) if err != nil { return nil, err } @@ -194,19 +209,26 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { } switch t.T { - case SliceTy: - if (*t.Elem).T == StringTy { - return forEachUnpack(t, output[begin:], 0, end) + case TupleTy: + if isDynamicType(t) { + begin, err := tuplePointsTo(index, output) + if err != nil { + return nil, err + } + return forTupleUnpack(t, output[begin:]) + } else { + return forTupleUnpack(t, output[index:]) } - return forEachUnpack(t, output, begin, end) + case SliceTy: + return forEachUnpack(t, output[begin:], 0, length) case ArrayTy: - if (*t.Elem).T == StringTy { + if isDynamicType(*t.Elem) { offset := int64(binary.BigEndian.Uint64(returnOutput[len(returnOutput)-8:])) return forEachUnpack(t, output[offset:], 0, t.Size) } - return forEachUnpack(t, output, index, t.Size) + return forEachUnpack(t, output[index:], 0, t.Size) case StringTy: // variable arrays are written at the end of the return bytes - return string(output[begin : begin+end]), nil + return string(output[begin : begin+length]), nil case IntTy, UintTy: return readInteger(t.T, t.Kind, returnOutput), nil case BoolTy: @@ -216,7 +238,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { case HashTy: return common.BytesToHash(returnOutput), nil case BytesTy: - return output[begin : begin+end], nil + return output[begin : begin+length], nil case FixedBytesTy: return readFixedBytes(t, returnOutput) case FunctionTy: @@ -257,3 +279,17 @@ func lengthPrefixPointsTo(index int, output []byte) (start int, length int, err length = int(lengthBig.Uint64()) return } + +// tuplePointsTo resolves the location reference for dynamic tuple. +func tuplePointsTo(index int, output []byte) (start int, err error) { + offset := big.NewInt(0).SetBytes(output[index : index+32]) + outputLen := big.NewInt(int64(len(output))) + + if offset.Cmp(big.NewInt(int64(len(output)))) > 0 { + return 0, fmt.Errorf("abi: cannot marshal in to go slice: offset %v would go over slice boundary (len=%v)", offset, outputLen) + } + if offset.BitLen() > 63 { + return 0, fmt.Errorf("abi offset larger than int64: %v", offset) + } + return int(offset.Uint64()), nil +} diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index 3873414c9f..5ece81ef28 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -173,7 +173,7 @@ var unpackTests = []unpackTest{ // multi dimensional, if these pass, all types that don't require length prefix should pass { def: `[{"type": "uint8[][]"}]`, - enc: "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000E0000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + enc: "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", want: [][]uint8{{1, 2}, {1, 2}}, }, { @@ -183,7 +183,7 @@ var unpackTests = []unpackTest{ }, { def: `[{"type": "uint8[][2]"}]`, - enc: "000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001", + enc: "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001", want: [2][]uint8{{1}, {1}}, }, { @@ -610,7 +610,18 @@ func TestMultiReturnWithStringSlice(t *testing.T) { t.Fatal(err) } buff := new(bytes.Buffer) - buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000001200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000008657468657265756d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b676f2d657468657265756d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000065")) + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) // output[0] offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000120")) // output[1] offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // output[0] length + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040")) // output[0][0] offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000080")) // output[0][1] offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000008")) // output[0][0] length + buff.Write(common.Hex2Bytes("657468657265756d000000000000000000000000000000000000000000000000")) // output[0][0] value + buff.Write(common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000b")) // output[0][1] length + buff.Write(common.Hex2Bytes("676f2d657468657265756d000000000000000000000000000000000000000000")) // output[0][1] value + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // output[1] length + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000064")) // output[1][0] value + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000065")) // output[1][1] value ret1, ret1Exp := new([]string), []string{"ethereum", "go-ethereum"} ret2, ret2Exp := new([]*big.Int), []*big.Int{big.NewInt(100), big.NewInt(101)} if err := abi.Unpack(&[]interface{}{ret1, ret2}, "multi", buff.Bytes()); err != nil { @@ -913,6 +924,108 @@ func TestUnmarshal(t *testing.T) { } } +func TestUnpackTuple(t *testing.T) { + const simpleTuple = `[{"name":"tuple","constant":false,"outputs":[{"type":"tuple","name":"ret","components":[{"type":"int256","name":"a"},{"type":"int256","name":"b"}]}]}]` + abi, err := JSON(strings.NewReader(simpleTuple)) + if err != nil { + t.Fatal(err) + } + buff := new(bytes.Buffer) + + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // ret[a] = 1 + buff.Write(common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) // ret[b] = -1 + + v := struct { + Ret struct { + A *big.Int + B *big.Int + } + }{Ret: struct { + A *big.Int + B *big.Int + }{new(big.Int), new(big.Int)}} + + err = abi.Unpack(&v, "tuple", buff.Bytes()) + if err != nil { + t.Error(err) + } else { + if v.Ret.A.Cmp(big.NewInt(1)) != 0 { + t.Errorf("unexpected value unpacked: want %x, got %x", 1, v.Ret.A) + } + if v.Ret.B.Cmp(big.NewInt(-1)) != 0 { + t.Errorf("unexpected value unpacked: want %x, got %x", v.Ret.B, -1) + } + } + + // Test nested tuple + const nestedTuple = `[{"name":"tuple","constant":false,"outputs":[ + {"type":"tuple","name":"s","components":[{"type":"uint256","name":"a"},{"type":"uint256[]","name":"b"},{"type":"tuple[]","name":"c","components":[{"name":"x", "type":"uint256"},{"name":"y","type":"uint256"}]}]}, + {"type":"tuple","name":"t","components":[{"name":"x", "type":"uint256"},{"name":"y","type":"uint256"}]}, + {"type":"uint256","name":"a"} + ]}]` + + abi, err = JSON(strings.NewReader(nestedTuple)) + if err != nil { + t.Fatal(err) + } + buff.Reset() + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000080")) // s offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000")) // t.X = 0 + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // t.Y = 1 + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // a = 1 + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // s.A = 1 + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000060")) // s.B offset + buff.Write(common.Hex2Bytes("00000000000000000000000000000000000000000000000000000000000000c0")) // s.C offset + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // s.B length + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // s.B[0] = 1 + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // s.B[0] = 2 + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // s.C length + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // s.C[0].X + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // s.C[0].Y + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002")) // s.C[1].X + buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // s.C[1].Y + + type T struct { + X *big.Int `abi:"x"` + Z *big.Int `abi:"y"` // Test whether the abi tag works. + } + + type S struct { + A *big.Int + B []*big.Int + C []T + } + + type Ret struct { + FieldS S `abi:"s"` + FieldT T `abi:"t"` + A *big.Int + } + var ret Ret + var expected = Ret{ + FieldS: S{ + A: big.NewInt(1), + B: []*big.Int{big.NewInt(1), big.NewInt(2)}, + C: []T{ + {big.NewInt(1), big.NewInt(2)}, + {big.NewInt(2), big.NewInt(1)}, + }, + }, + FieldT: T{ + big.NewInt(0), big.NewInt(1), + }, + A: big.NewInt(1), + } + + err = abi.Unpack(&ret, "tuple", buff.Bytes()) + if err != nil { + t.Error(err) + } + if reflect.DeepEqual(ret, expected) { + t.Error("unexpected unpack value") + } +} + func TestOOMMaliciousInput(t *testing.T) { oomTests := []unpackTest{ { From 7240f4d800d3ffb97cf23a27d13746428d761cec Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 10 Jan 2019 12:33:51 +0100 Subject: [PATCH 11/14] swarm/network: Rename minproxbinsize, add as member of simulation (#18408) * swarm/network: Rename minproxbinsize, add as member of simulation * swarm/network: Deactivate WaitTillHealthy, unreliable pending suggestpeer --- swarm/network/kademlia.go | 70 ++++++++++--------- swarm/network/kademlia_test.go | 20 +++--- swarm/network/networkid_test.go | 2 +- swarm/network/simulation/example_test.go | 13 ++-- swarm/network/simulation/kademlia.go | 4 +- swarm/network/simulation/kademlia_test.go | 3 +- swarm/network/simulation/simulation.go | 21 +++--- .../simulations/discovery/discovery_test.go | 10 +-- swarm/network/simulations/overlay.go | 2 +- swarm/network/stream/delivery_test.go | 4 +- swarm/network/stream/intervals_test.go | 2 +- .../network/stream/snapshot_retrieval_test.go | 4 +- swarm/network/stream/snapshot_sync_test.go | 8 +-- swarm/network/stream/syncer_test.go | 2 +- .../visualized_snapshot_sync_sim_test.go | 2 +- swarm/network_test.go | 2 +- swarm/pss/client/client_test.go | 2 +- swarm/pss/notify/notify_test.go | 2 +- swarm/pss/pss_test.go | 4 +- 19 files changed, 93 insertions(+), 84 deletions(-) diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 3214e151d1..341ed20b28 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -54,13 +54,13 @@ var Pof = pot.DefaultPof(256) // KadParams holds the config params for Kademlia type KadParams struct { // adjustable parameters - MaxProxDisplay int // number of rows the table shows - MinProxBinSize int // nearest neighbour core minimum cardinality - MinBinSize int // minimum number of peers in a row - MaxBinSize int // maximum number of peers in a row before pruning - RetryInterval int64 // initial interval before a peer is first redialed - RetryExponent int // exponent to multiply retry intervals with - MaxRetries int // maximum number of redial attempts + MaxProxDisplay int // number of rows the table shows + NeighbourhoodSize int // nearest neighbour core minimum cardinality + MinBinSize int // minimum number of peers in a row + MaxBinSize int // maximum number of peers in a row before pruning + RetryInterval int64 // initial interval before a peer is first redialed + RetryExponent int // exponent to multiply retry intervals with + MaxRetries int // maximum number of redial attempts // function to sanction or prevent suggesting a peer Reachable func(*BzzAddr) bool `json:"-"` } @@ -68,13 +68,13 @@ type KadParams struct { // NewKadParams returns a params struct with default values func NewKadParams() *KadParams { return &KadParams{ - MaxProxDisplay: 16, - MinProxBinSize: 2, - MinBinSize: 2, - MaxBinSize: 4, - RetryInterval: 4200000000, // 4.2 sec - MaxRetries: 42, - RetryExponent: 2, + MaxProxDisplay: 16, + NeighbourhoodSize: 2, + MinBinSize: 2, + MaxBinSize: 4, + RetryInterval: 4200000000, // 4.2 sec + MaxRetries: 42, + RetryExponent: 2, } } @@ -175,7 +175,7 @@ func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { k.lock.Lock() defer k.lock.Unlock() minsize := k.MinBinSize - depth := depthForPot(k.conns, k.MinProxBinSize, k.base) + depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) // if there is a callable neighbour within the current proxBin, connect // this makes sure nearest neighbour set is fully connected var ppo int @@ -306,7 +306,7 @@ func (k *Kademlia) sendNeighbourhoodDepthChange() { // It provides signaling of neighbourhood depth change. // This part of the code is sending new neighbourhood depth to nDepthC if that condition is met. if k.nDepthC != nil { - nDepth := depthForPot(k.conns, k.MinProxBinSize, k.base) + nDepth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) if nDepth != k.nDepth { k.nDepth = nDepth k.nDepthC <- nDepth @@ -366,7 +366,7 @@ func (k *Kademlia) EachBin(base []byte, pof pot.Pof, o int, eachBinFunc func(con var startPo int var endPo int - kadDepth := depthForPot(k.conns, k.MinProxBinSize, k.base) + kadDepth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) k.conns.EachBin(base, Pof, o, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { if startPo > 0 && endPo != k.MaxProxDisplay { @@ -432,15 +432,15 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int) bool) { func (k *Kademlia) NeighbourhoodDepth() (depth int) { k.lock.RLock() defer k.lock.RUnlock() - return depthForPot(k.conns, k.MinProxBinSize, k.base) + return depthForPot(k.conns, k.NeighbourhoodSize, k.base) } // depthForPot returns the proximity order that defines the distance of -// the nearest neighbour set with cardinality >= MinProxBinSize -// if there is altogether less than MinProxBinSize peers it returns 0 +// the nearest neighbour set with cardinality >= NeighbourhoodSize +// if there is altogether less than NeighbourhoodSize peers it returns 0 // caller must hold the lock -func depthForPot(p *pot.Pot, minProxBinSize int, pivotAddr []byte) (depth int) { - if p.Size() <= minProxBinSize { +func depthForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int) { + if p.Size() <= neighbourhoodSize { return 0 } @@ -448,7 +448,7 @@ func depthForPot(p *pot.Pot, minProxBinSize int, pivotAddr []byte) (depth int) { var size int // determining the depth is a two-step process - // first we find the proximity bin of the shallowest of the MinProxBinSize peers + // first we find the proximity bin of the shallowest of the NeighbourhoodSize peers // the numeric value of depth cannot be higher than this var maxDepth int @@ -461,7 +461,7 @@ func depthForPot(p *pot.Pot, minProxBinSize int, pivotAddr []byte) (depth int) { // this means we have all nn-peers. // depth is by default set to the bin of the farthest nn-peer - if size == minProxBinSize { + if size == neighbourhoodSize { maxDepth = i return false } @@ -538,12 +538,12 @@ func (k *Kademlia) string() string { rows = append(rows, "=========================================================================") rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %x", time.Now().UTC().Format(time.UnixDate), k.BaseAddr()[:3])) - rows = append(rows, fmt.Sprintf("population: %d (%d), MinProxBinSize: %d, MinBinSize: %d, MaxBinSize: %d", k.conns.Size(), k.addrs.Size(), k.MinProxBinSize, k.MinBinSize, k.MaxBinSize)) + rows = append(rows, fmt.Sprintf("population: %d (%d), NeighbourhoodSize: %d, MinBinSize: %d, MaxBinSize: %d", k.conns.Size(), k.addrs.Size(), k.NeighbourhoodSize, k.MinBinSize, k.MaxBinSize)) liverows := make([]string, k.MaxProxDisplay) peersrows := make([]string, k.MaxProxDisplay) - depth := depthForPot(k.conns, k.MinProxBinSize, k.base) + depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) rest := k.conns.Size() k.conns.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val, i int) bool) bool) bool { var rowlen int @@ -611,10 +611,10 @@ type PeerPot struct { // NewPeerPotMap creates a map of pot record of *BzzAddr with keys // as hexadecimal representations of the address. -// the MinProxBinSize of the passed kademlia is used +// the NeighbourhoodSize of the passed kademlia is used // used for testing only // TODO move to separate testing tools file -func NewPeerPotMap(minProxBinSize int, addrs [][]byte) map[string]*PeerPot { +func NewPeerPotMap(neighbourhoodSize int, addrs [][]byte) map[string]*PeerPot { // create a table of all nodes for health check np := pot.NewPot(nil, 0) @@ -628,7 +628,7 @@ func NewPeerPotMap(minProxBinSize int, addrs [][]byte) map[string]*PeerPot { for i, a := range addrs { // actual kademlia depth - depth := depthForPot(np, minProxBinSize, a) + depth := depthForPot(np, neighbourhoodSize, a) // all nn-peers var nns [][]byte @@ -670,7 +670,7 @@ func (k *Kademlia) saturation() int { return prev == po && size >= k.MinBinSize }) // TODO evaluate whether this check cannot just as well be done within the eachbin - depth := depthForPot(k.conns, k.MinProxBinSize, k.base) + depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) if depth < prev { return depth } @@ -683,7 +683,7 @@ func (k *Kademlia) saturation() int { // TODO move to separate testing tools file func (k *Kademlia) knowNeighbours(addrs [][]byte) (got bool, n int, missing [][]byte) { pm := make(map[string]bool) - depth := depthForPot(k.conns, k.MinProxBinSize, k.base) + depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) // create a map with all peers at depth and deeper known in the kademlia k.eachAddr(nil, 255, func(p *BzzAddr, po int) bool { // in order deepest to shallowest compared to the kademlia base address @@ -719,7 +719,11 @@ func (k *Kademlia) knowNeighbours(addrs [][]byte) (got bool, n int, missing [][] // It is used in Healthy function for testing only func (k *Kademlia) connectedNeighbours(peers [][]byte) (got bool, n int, missing [][]byte) { pm := make(map[string]bool) - depth := depthForPot(k.conns, k.MinProxBinSize, k.base) + + // create a map with all peers at depth and deeper that are connected in the kademlia + // in order deepest to shallowest compared to the kademlia base address + // all bins (except self) are included (0 <= bin <= 255) + depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) k.eachConn(nil, 255, func(p *Peer, po int) bool { if po < depth { return false @@ -772,7 +776,7 @@ func (k *Kademlia) Healthy(pp *PeerPot) *Health { defer k.lock.RUnlock() gotnn, countgotnn, culpritsgotnn := k.connectedNeighbours(pp.NNSet) knownn, countknownn, culpritsknownn := k.knowNeighbours(pp.NNSet) - depth := depthForPot(k.conns, k.MinProxBinSize, k.base) + depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) saturated := k.saturation() < depth log.Trace(fmt.Sprintf("%08x: healthy: knowNNs: %v, gotNNs: %v, saturated: %v\n", k.base, knownn, gotnn, saturated)) return &Health{ diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index 8d320c1728..c1a26f6122 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -45,7 +45,7 @@ func newTestKademliaParams() *KadParams { params := NewKadParams() // TODO why is this 1? params.MinBinSize = 1 - params.MinProxBinSize = 2 + params.NeighbourhoodSize = 2 return params } @@ -87,7 +87,7 @@ func Register(k *Kademlia, regs ...string) { // empty bins above the farthest "nearest neighbor-peer" then // the depth should be set at the farthest of those empty bins // -// TODO: Make test adapt to change in MinProxBinSize +// TODO: Make test adapt to change in NeighbourhoodSize func TestNeighbourhoodDepth(t *testing.T) { baseAddressBytes := RandomAddr().OAddr kad := NewKademlia(baseAddressBytes, NewKadParams()) @@ -237,7 +237,7 @@ func assertHealth(t *testing.T, k *Kademlia, expectHealthy bool, expectSaturatio return true }) - pp := NewPeerPotMap(k.MinProxBinSize, addrs) + pp := NewPeerPotMap(k.NeighbourhoodSize, addrs) healthParams := k.Healthy(pp[kid]) // definition of health, all conditions but be true: @@ -605,7 +605,7 @@ func TestKademliaHiveString(t *testing.T) { Register(k, "10000000", "10000001") k.MaxProxDisplay = 8 h := k.String() - expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), MinProxBinSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000 0 | 2 8100 (0) 8000 (0)\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" + expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), NeighbourhoodSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000 0 | 2 8100 (0) 8000 (0)\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" if expH[104:] != h[104:] { t.Fatalf("incorrect hive output. expected %v, got %v", expH, h) } @@ -636,7 +636,7 @@ func testKademliaCase(t *testing.T, pivotAddr string, addrs ...string) { } } - ppmap := NewPeerPotMap(k.MinProxBinSize, byteAddrs) + ppmap := NewPeerPotMap(k.NeighbourhoodSize, byteAddrs) pp := ppmap[pivotAddr] @@ -662,7 +662,7 @@ in higher level tests for streaming. They were generated randomly. ========================================================================= Mon Apr 9 12:18:24 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 7efef1 -population: 9 (49), MinProxBinSize: 2, MinBinSize: 2, MaxBinSize: 4 +population: 9 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 000 2 d7e5 ec56 | 18 ec56 (0) d7e5 (0) d9e0 (0) c735 (0) 001 2 18f1 3176 | 14 18f1 (0) 10bb (0) 10d1 (0) 0421 (0) 002 2 52aa 47cd | 11 52aa (0) 51d9 (0) 5161 (0) 5130 (0) @@ -745,7 +745,7 @@ in higher level tests for streaming. They were generated randomly. ========================================================================= Mon Apr 9 18:43:48 UTC 2018 KΛÐΞMLIΛ hive: queen's address: bc7f3b -population: 9 (49), MinProxBinSize: 2, MinBinSize: 2, MaxBinSize: 4 +population: 9 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 000 2 0f49 67ff | 28 0f49 (0) 0211 (0) 07b2 (0) 0703 (0) 001 2 e84b f3a4 | 13 f3a4 (0) e84b (0) e58b (0) e60b (0) 002 1 8dba | 1 8dba (0) @@ -779,7 +779,7 @@ in higher level tests for streaming. They were generated randomly. ========================================================================= Mon Apr 9 19:04:35 UTC 2018 KΛÐΞMLIΛ hive: queen's address: b4822e -population: 8 (49), MinProxBinSize: 2, MinBinSize: 2, MaxBinSize: 4 +population: 8 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 000 2 786c 774b | 29 774b (0) 786c (0) 7a79 (0) 7d2f (0) 001 2 d9de cf19 | 10 cf19 (0) d9de (0) d2ff (0) d2a2 (0) 002 2 8ca1 8d74 | 5 8d74 (0) 8ca1 (0) 9793 (0) 9f51 (0) @@ -813,7 +813,7 @@ in higher level tests for streaming. They were generated randomly. ========================================================================= Mon Apr 9 19:16:25 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 9a90fe -population: 8 (49), MinProxBinSize: 2, MinBinSize: 2, MaxBinSize: 4 +population: 8 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 000 2 72ef 4e6c | 24 0b1e (0) 0d66 (0) 17f5 (0) 17e8 (0) 001 2 fc2b fa47 | 13 fa47 (0) fc2b (0) fffd (0) ecef (0) 002 2 b847 afa8 | 6 afa8 (0) ad77 (0) bb7c (0) b847 (0) @@ -848,7 +848,7 @@ in higher level tests for streaming. They were generated randomly. ========================================================================= Mon Apr 9 19:25:18 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 5dd5c7 -population: 13 (49), MinProxBinSize: 2, MinBinSize: 2, MaxBinSize: 4 +population: 13 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 000 2 e528 fad0 | 22 fad0 (0) e528 (0) e3bb (0) ed13 (0) 001 3 3f30 18e0 1dd3 | 7 3f30 (0) 23db (0) 10b6 (0) 18e0 (0) 002 4 7c54 7804 61e4 60f9 | 10 61e4 (0) 60f9 (0) 636c (0) 7186 (0) diff --git a/swarm/network/networkid_test.go b/swarm/network/networkid_test.go index 3cd683e1c9..99890118fc 100644 --- a/swarm/network/networkid_test.go +++ b/swarm/network/networkid_test.go @@ -188,7 +188,7 @@ func newServices() adapters.Services { return k } params := NewKadParams() - params.MinProxBinSize = 2 + params.NeighbourhoodSize = 2 params.MaxBinSize = 3 params.MinBinSize = 1 params.MaxRetries = 1000 diff --git a/swarm/network/simulation/example_test.go b/swarm/network/simulation/example_test.go index e9a360dfd4..9d14929795 100644 --- a/swarm/network/simulation/example_test.go +++ b/swarm/network/simulation/example_test.go @@ -18,8 +18,14 @@ package simulation_test import ( "context" + "fmt" + "sync" + "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" ) @@ -28,10 +34,6 @@ import ( // all nodes have the their Kademlias healthy. func ExampleSimulation_WaitTillHealthy() { - log.Error("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") - - /* Commented out to avoid go vet errors/warnings - sim := simulation.New(map[string]simulation.ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { addr := network.NewAddr(ctx.Config.Node()) @@ -59,7 +61,7 @@ func ExampleSimulation_WaitTillHealthy() { ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() - ill, err := sim.WaitTillHealthy(ctx, 2) + ill, err := sim.WaitTillHealthy(ctx) if err != nil { // inspect the latest detected not healthy kademlias for id, kad := range ill { @@ -71,7 +73,6 @@ func ExampleSimulation_WaitTillHealthy() { // continue with the test - */ } // Watch all peer events in the simulation network, buy receiving from a channel. diff --git a/swarm/network/simulation/kademlia.go b/swarm/network/simulation/kademlia.go index ebec468f14..6d8d0e0a2c 100644 --- a/swarm/network/simulation/kademlia.go +++ b/swarm/network/simulation/kademlia.go @@ -34,7 +34,7 @@ var BucketKeyKademlia BucketKey = "kademlia" // WaitTillHealthy is blocking until the health of all kademlias is true. // If error is not nil, a map of kademlia that was found not healthy is returned. // TODO: Check correctness since change in kademlia depth calculation logic -func (s *Simulation) WaitTillHealthy(ctx context.Context, kadMinProxSize int) (ill map[enode.ID]*network.Kademlia, err error) { +func (s *Simulation) WaitTillHealthy(ctx context.Context) (ill map[enode.ID]*network.Kademlia, err error) { // Prepare PeerPot map for checking Kademlia health var ppmap map[string]*network.PeerPot kademlias := s.kademlias() @@ -43,7 +43,7 @@ func (s *Simulation) WaitTillHealthy(ctx context.Context, kadMinProxSize int) (i for _, k := range kademlias { addrs = append(addrs, k.BaseAddr()) } - ppmap = network.NewPeerPotMap(kadMinProxSize, addrs) + ppmap = network.NewPeerPotMap(s.neighbourhoodSize, addrs) // Wait for healthy Kademlia on every node before checking files ticker := time.NewTicker(200 * time.Millisecond) diff --git a/swarm/network/simulation/kademlia_test.go b/swarm/network/simulation/kademlia_test.go index f02b0e5417..36b244d3d0 100644 --- a/swarm/network/simulation/kademlia_test.go +++ b/swarm/network/simulation/kademlia_test.go @@ -28,6 +28,7 @@ import ( ) func TestWaitTillHealthy(t *testing.T) { + t.Skip("WaitTillHealthy depends on discovery, which relies on a reliable SuggestPeer, which is not reliable") sim := New(map[string]ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { @@ -54,7 +55,7 @@ func TestWaitTillHealthy(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() - ill, err := sim.WaitTillHealthy(ctx, 2) + ill, err := sim.WaitTillHealthy(ctx) if err != nil { for id, kad := range ill { t.Log("Node", id) diff --git a/swarm/network/simulation/simulation.go b/swarm/network/simulation/simulation.go index 106eeb71e9..13c5b1c575 100644 --- a/swarm/network/simulation/simulation.go +++ b/swarm/network/simulation/simulation.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/network" ) // Common errors that are returned by functions in this package. @@ -42,13 +43,14 @@ type Simulation struct { // of p2p/simulations.Network. Net *simulations.Network - serviceNames []string - cleanupFuncs []func() - buckets map[enode.ID]*sync.Map - pivotNodeID *enode.ID - shutdownWG sync.WaitGroup - done chan struct{} - mu sync.RWMutex + serviceNames []string + cleanupFuncs []func() + buckets map[enode.ID]*sync.Map + pivotNodeID *enode.ID + shutdownWG sync.WaitGroup + done chan struct{} + mu sync.RWMutex + neighbourhoodSize int httpSrv *http.Server //attach a HTTP server via SimulationOptions handler *simulations.Server //HTTP handler for the server @@ -72,8 +74,9 @@ type ServiceFunc func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Se // which is used to start node.Service returned by ServiceFunc. func New(services map[string]ServiceFunc) (s *Simulation) { s = &Simulation{ - buckets: make(map[enode.ID]*sync.Map), - done: make(chan struct{}), + buckets: make(map[enode.ID]*sync.Map), + done: make(chan struct{}), + neighbourhoodSize: network.NewKadParams().NeighbourhoodSize, } adapterServices := make(map[string]adapters.ServiceFunc, len(services)) diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index bd86865220..e5121c4775 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -46,7 +46,7 @@ import ( // serviceName is used with the exec adapter so the exec'd binary knows which // service to execute const serviceName = "discovery" -const testMinProxBinSize = 2 +const testNeighbourhoodSize = 2 const discoveryPersistenceDatadir = "discovery_persistence_test_store" var discoveryPersistencePath = path.Join(os.TempDir(), discoveryPersistenceDatadir) @@ -268,7 +268,7 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul wg.Wait() log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) // construct the peer pot, so that kademlia health can be checked - ppmap := network.NewPeerPotMap(network.NewKadParams().MinProxBinSize, addrs) + ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) check := func(ctx context.Context, id enode.ID) (bool, error) { select { case <-ctx.Done(): @@ -404,7 +404,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt } healthy := &network.Health{} addr := id.String() - ppmap := network.NewPeerPotMap(network.NewKadParams().MinProxBinSize, addrs) + ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) if err := client.Call(&healthy, "hive_healthy", ppmap); err != nil { return fmt.Errorf("error getting node health: %s", err) } @@ -492,7 +492,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt return false, fmt.Errorf("error getting node client: %s", err) } healthy := &network.Health{} - ppmap := network.NewPeerPotMap(network.NewKadParams().MinProxBinSize, addrs) + ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) if err := client.Call(&healthy, "hive_healthy", ppmap); err != nil { return false, fmt.Errorf("error getting node health: %s", err) @@ -566,7 +566,7 @@ func newService(ctx *adapters.ServiceContext) (node.Service, error) { addr := network.NewAddr(ctx.Config.Node()) kp := network.NewKadParams() - kp.MinProxBinSize = testMinProxBinSize + kp.NeighbourhoodSize = testNeighbourhoodSize if ctx.Config.Reachable != nil { kp.Reachable = func(o *network.BzzAddr) bool { diff --git a/swarm/network/simulations/overlay.go b/swarm/network/simulations/overlay.go index 284ae6398a..63938809e4 100644 --- a/swarm/network/simulations/overlay.go +++ b/swarm/network/simulations/overlay.go @@ -86,7 +86,7 @@ func (s *Simulation) NewService(ctx *adapters.ServiceContext) (node.Service, err addr := network.NewAddr(node) kp := network.NewKadParams() - kp.MinProxBinSize = 2 + kp.NeighbourhoodSize = 2 kp.MaxBinSize = 4 kp.MinBinSize = 1 kp.MaxRetries = 1000 diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index c65e1386d0..70d3829b3f 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -544,7 +544,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) log.Debug("Waiting for kademlia") // TODO this does not seem to be correct usage of the function, as the simulation may have no kademlias - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { return err } @@ -704,7 +704,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b } netStore := item.(*storage.NetStore) - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { return err } diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 121758c187..8f2bed9d64 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -115,7 +115,7 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { t.Fatal(err) } diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index a85d723297..d345ac8d02 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -197,7 +197,7 @@ func runFileRetrievalTest(nodeCount int) error { if err != nil { return err } - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { return err } @@ -287,7 +287,7 @@ func runRetrievalTest(chunkCount int, nodeCount int) error { if err != nil { return err } - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { return err } diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 7a883644b9..4e4497ccd6 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -204,7 +204,7 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) defer cancelSimRun() - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { t.Fatal(err) } @@ -391,7 +391,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) return err } - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { return err } @@ -471,7 +471,7 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) conf.hashes = append(conf.hashes, hashes...) mapKeysToNodes(conf) - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { return err } @@ -563,7 +563,7 @@ func mapKeysToNodes(conf *synctestConfig) { np, _, _ = pot.Add(np, a, pof) } - ppmap := network.NewPeerPotMap(network.NewKadParams().MinProxBinSize, conf.addrs) + ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, conf.addrs) //for each address, run EachNeighbour on the chunk hashes pot to identify closest nodes log.Trace(fmt.Sprintf("Generated hash chunk(s): %v", conf.hashes)) diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 09152ebfe1..014ec9a982 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -189,7 +189,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p } } // here we distribute chunks of a random file into stores 1...nodes - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { return err } diff --git a/swarm/network/stream/visualized_snapshot_sync_sim_test.go b/swarm/network/stream/visualized_snapshot_sync_sim_test.go index 638eae6e3d..18b4c8fb0b 100644 --- a/swarm/network/stream/visualized_snapshot_sync_sim_test.go +++ b/swarm/network/stream/visualized_snapshot_sync_sim_test.go @@ -72,7 +72,7 @@ func setupSim(serviceMap map[string]simulation.ServiceFunc) (int, int, *simulati func watchSim(sim *simulation.Simulation) (context.Context, context.CancelFunc) { ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute) - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { panic(err) } diff --git a/swarm/network_test.go b/swarm/network_test.go index 8a162a219e..71d4b8f16a 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -353,7 +353,7 @@ func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwa } if *waitKademlia { - if _, err := sim.WaitTillHealthy(ctx, 2); err != nil { + if _, err := sim.WaitTillHealthy(ctx); err != nil { return err } } diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index 8f2f0e8059..0d6788d679 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -238,7 +238,7 @@ func newServices() adapters.Services { return k } params := network.NewKadParams() - params.MinProxBinSize = 2 + params.NeighbourhoodSize = 2 params.MaxBinSize = 3 params.MinBinSize = 1 params.MaxRetries = 1000 diff --git a/swarm/pss/notify/notify_test.go b/swarm/pss/notify/notify_test.go index 6100195b09..5c29f68e0f 100644 --- a/swarm/pss/notify/notify_test.go +++ b/swarm/pss/notify/notify_test.go @@ -209,7 +209,7 @@ func newServices(allowRaw bool) adapters.Services { return k } params := network.NewKadParams() - params.MinProxBinSize = 2 + params.NeighbourhoodSize = 2 params.MaxBinSize = 3 params.MinBinSize = 1 params.MaxRetries = 1000 diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index b0753ad178..46daa46741 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -1965,7 +1965,7 @@ func newServices(allowRaw bool) adapters.Services { return k } params := network.NewKadParams() - params.MinProxBinSize = 2 + params.NeighbourhoodSize = 2 params.MaxBinSize = 3 params.MinBinSize = 1 params.MaxRetries = 1000 @@ -2045,7 +2045,7 @@ func newTestPss(privkey *ecdsa.PrivateKey, kad *network.Kademlia, ppextra *PssPa // set up routing if kademlia is not passed to us if kad == nil { kp := network.NewKadParams() - kp.MinProxBinSize = 3 + kp.NeighbourhoodSize = 3 kad = network.NewKademlia(nid[:], kp) } From 38cce9ac333d674616047be14c270d7cfbd43641 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Thu, 10 Jan 2019 16:27:54 +0100 Subject: [PATCH 12/14] accounts/abi: Extra slice tests (#18424) Co-authored-by: weimumu <934657014@qq.com> --- accounts/abi/unpack_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index 5ece81ef28..ff88be3d30 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -176,6 +176,11 @@ var unpackTests = []unpackTest{ enc: "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", want: [][]uint8{{1, 2}, {1, 2}}, }, + { + def: `[{"type": "uint8[][]"}]`, + enc: "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003", + want: [][]uint8{{1, 2}, {1, 2, 3}}, + }, { def: `[{"type": "uint8[2][2]"}]`, enc: "0000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", @@ -251,6 +256,16 @@ var unpackTests = []unpackTest{ enc: "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000008457468657265756d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000b676f2d657468657265756d000000000000000000000000000000000000000000", want: []string{"Ethereum", "go-ethereum"}, }, + { + def: `[{"type": "bytes[]"}]`, + enc: "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000003f0f0f000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003f0f0f00000000000000000000000000000000000000000000000000000000000", + want: [][]byte{{0xf0, 0xf0, 0xf0}, {0xf0, 0xf0, 0xf0}}, + }, + { + def: `[{"type": "uint256[2][][]"}]`, + enc: "00000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000c8000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000003e80000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000c8000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000003e8", + want: [][][2]*big.Int{{{big.NewInt(1), big.NewInt(200)}, {big.NewInt(1), big.NewInt(1000)}}, {{big.NewInt(1), big.NewInt(200)}, {big.NewInt(1), big.NewInt(1000)}}}, + }, { def: `[{"type": "int8[]"}]`, enc: "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", From 2eb838ed9776c9c3ec922e1116a5d50babda31c5 Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Fri, 11 Jan 2019 10:23:45 +0100 Subject: [PATCH 13/14] p2p/simulations: eliminate concept of pivot (#18426) --- p2p/simulations/adapters/inproc.go | 14 ------ p2p/simulations/connect.go | 52 ++------------------- p2p/simulations/connect_test.go | 65 +++++++++----------------- p2p/simulations/network.go | 2 - swarm/network/simulation/node.go | 21 +-------- swarm/network/simulation/node_test.go | 39 ---------------- swarm/network/simulation/simulation.go | 1 - 7 files changed, 28 insertions(+), 166 deletions(-) diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index d122cd6487..eada9579ed 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -351,17 +351,3 @@ func (sn *SimNode) NodeInfo() *p2p.NodeInfo { } return server.NodeInfo() } - -func setSocketBuffer(conn net.Conn, socketReadBuffer int, socketWriteBuffer int) error { - if v, ok := conn.(*net.UnixConn); ok { - err := v.SetReadBuffer(socketReadBuffer) - if err != nil { - return err - } - err = v.SetWriteBuffer(socketWriteBuffer) - if err != nil { - return err - } - } - return nil -} diff --git a/p2p/simulations/connect.go b/p2p/simulations/connect.go index 606dda0567..bb7e7999a1 100644 --- a/p2p/simulations/connect.go +++ b/p2p/simulations/connect.go @@ -25,21 +25,8 @@ import ( var ( ErrNodeNotFound = errors.New("node not found") - ErrNoPivotNode = errors.New("no pivot node set") ) -// ConnectToPivotNode connects the node with provided NodeID -// to the pivot node, already set by Network.SetPivotNode method. -// It is useful when constructing a star network topology -// when Network adds and removes nodes dynamically. -func (net *Network) ConnectToPivotNode(id enode.ID) (err error) { - pivot := net.GetPivotNode() - if pivot == nil { - return ErrNoPivotNode - } - return net.connect(pivot.ID(), id) -} - // ConnectToLastNode connects the node with provided NodeID // to the last node that is up, and avoiding connection to self. // It is useful when constructing a chain network topology @@ -115,35 +102,23 @@ func (net *Network) ConnectNodesRing(ids []enode.ID) (err error) { return net.connect(ids[l-1], ids[0]) } -// ConnectNodesStar connects all nodes in a star topology -// with the center at provided NodeID. +// ConnectNodesStar connects all nodes into a star topology // If ids argument is nil, all nodes that are up will be connected. -func (net *Network) ConnectNodesStar(pivot enode.ID, ids []enode.ID) (err error) { +func (net *Network) ConnectNodesStar(ids []enode.ID, center enode.ID) (err error) { if ids == nil { ids = net.getUpNodeIDs() } for _, id := range ids { - if pivot == id { + if center == id { continue } - if err := net.connect(pivot, id); err != nil { + if err := net.connect(center, id); err != nil { return err } } return nil } -// ConnectNodesStarPivot connects all nodes in a star topology -// with the center at already set pivot node. -// If ids argument is nil, all nodes that are up will be connected. -func (net *Network) ConnectNodesStarPivot(ids []enode.ID) (err error) { - pivot := net.GetPivotNode() - if pivot == nil { - return ErrNoPivotNode - } - return net.ConnectNodesStar(pivot.ID(), ids) -} - // connect connects two nodes but ignores already connected error. func (net *Network) connect(oneID, otherID enode.ID) error { return ignoreAlreadyConnectedErr(net.Connect(oneID, otherID)) @@ -155,22 +130,3 @@ func ignoreAlreadyConnectedErr(err error) error { } return err } - -// SetPivotNode sets the NodeID of the network's pivot node. -// Pivot node is just a specific node that should be treated -// differently then other nodes in test. SetPivotNode and -// GetPivotNode are just a convenient functions to set and -// retrieve it. -func (net *Network) SetPivotNode(id enode.ID) { - net.lock.Lock() - defer net.lock.Unlock() - net.pivotNodeID = id -} - -// GetPivotNode returns NodeID of the pivot node set by -// Network.SetPivotNode method. -func (net *Network) GetPivotNode() (node *Node) { - net.lock.RLock() - defer net.lock.RUnlock() - return net.getNode(net.pivotNodeID) -} diff --git a/p2p/simulations/connect_test.go b/p2p/simulations/connect_test.go index fd2bf5c845..32d18347d8 100644 --- a/p2p/simulations/connect_test.go +++ b/p2p/simulations/connect_test.go @@ -58,24 +58,6 @@ func newTestNetwork(t *testing.T, nodeCount int) (*Network, []enode.ID) { return network, ids } -func TestConnectToPivotNode(t *testing.T) { - net, ids := newTestNetwork(t, 2) - defer net.Shutdown() - - pivot := ids[0] - net.SetPivotNode(pivot) - - other := ids[1] - err := net.ConnectToPivotNode(other) - if err != nil { - t.Fatal(err) - } - - if net.GetConn(pivot, other) == nil { - t.Error("pivot and the other node are not connected") - } -} - func TestConnectToLastNode(t *testing.T) { net, ids := newTestNetwork(t, 10) defer net.Shutdown() @@ -125,15 +107,30 @@ func TestConnectToRandomNode(t *testing.T) { } func TestConnectNodesFull(t *testing.T) { - net, ids := newTestNetwork(t, 12) - defer net.Shutdown() - - err := net.ConnectNodesFull(ids) - if err != nil { - t.Fatal(err) + tests := []struct { + name string + nodeCount int + }{ + {name: "no node", nodeCount: 0}, + {name: "single node", nodeCount: 1}, + {name: "2 nodes", nodeCount: 2}, + {name: "3 nodes", nodeCount: 3}, + {name: "even number of nodes", nodeCount: 12}, + {name: "odd number of nodes", nodeCount: 13}, } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + net, ids := newTestNetwork(t, test.nodeCount) + defer net.Shutdown() - VerifyFull(t, net, ids) + err := net.ConnectNodesFull(ids) + if err != nil { + t.Fatal(err) + } + + VerifyFull(t, net, ids) + }) + } } func TestConnectNodesChain(t *testing.T) { @@ -166,23 +163,7 @@ func TestConnectNodesStar(t *testing.T) { pivotIndex := 2 - err := net.ConnectNodesStar(ids[pivotIndex], ids) - if err != nil { - t.Fatal(err) - } - - VerifyStar(t, net, ids, pivotIndex) -} - -func TestConnectNodesStarPivot(t *testing.T) { - net, ids := newTestNetwork(t, 10) - defer net.Shutdown() - - pivotIndex := 4 - - net.SetPivotNode(ids[pivotIndex]) - - err := net.ConnectNodesStarPivot(ids) + err := net.ConnectNodesStar(ids, ids[pivotIndex]) if err != nil { t.Fatal(err) } diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index a711fd78e3..86f7dc9bef 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -58,8 +58,6 @@ type Network struct { Conns []*Conn `json:"conns"` connMap map[string]int - pivotNodeID enode.ID - nodeAdapter adapters.NodeAdapter events event.Feed lock sync.RWMutex diff --git a/swarm/network/simulation/node.go b/swarm/network/simulation/node.go index e8d4d6d948..08eb835247 100644 --- a/swarm/network/simulation/node.go +++ b/swarm/network/simulation/node.go @@ -188,7 +188,7 @@ func (s *Simulation) AddNodesAndConnectStar(count int, opts ...AddNodeOption) (i if err != nil { return nil, err } - err = s.Net.ConnectNodesStar(ids[0], ids[1:]) + err = s.Net.ConnectNodesStar(ids[1:], ids[0]) if err != nil { return nil, err } @@ -241,25 +241,6 @@ func (s *Simulation) UploadSnapshot(snapshotFile string, opts ...AddNodeOption) return nil } -// SetPivotNode sets the NodeID of the network's pivot node. -// Pivot node is just a specific node that should be treated -// differently then other nodes in test. SetPivotNode and -// PivotNodeID are just a convenient functions to set and -// retrieve it. -func (s *Simulation) SetPivotNode(id enode.ID) { - s.mu.Lock() - defer s.mu.Unlock() - s.pivotNodeID = &id -} - -// PivotNodeID returns NodeID of the pivot node set by -// Simulation.SetPivotNode method. -func (s *Simulation) PivotNodeID() (id *enode.ID) { - s.mu.Lock() - defer s.mu.Unlock() - return s.pivotNodeID -} - // StartNode starts a node by NodeID. func (s *Simulation) StartNode(id enode.ID) (err error) { return s.Net.Start(id) diff --git a/swarm/network/simulation/node_test.go b/swarm/network/simulation/node_test.go index 8da32cf37f..dc9189c911 100644 --- a/swarm/network/simulation/node_test.go +++ b/swarm/network/simulation/node_test.go @@ -314,45 +314,6 @@ func TestUploadSnapshot(t *testing.T) { log.Debug("Done.") } -func TestPivotNode(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - id, err := sim.AddNode() - if err != nil { - t.Fatal(err) - } - - id2, err := sim.AddNode() - if err != nil { - t.Fatal(err) - } - - if sim.PivotNodeID() != nil { - t.Error("expected no pivot node") - } - - sim.SetPivotNode(id) - - pid := sim.PivotNodeID() - - if pid == nil { - t.Error("pivot node not set") - } else if *pid != id { - t.Errorf("expected pivot node %s, got %s", id, *pid) - } - - sim.SetPivotNode(id2) - - pid = sim.PivotNodeID() - - if pid == nil { - t.Error("pivot node not set") - } else if *pid != id2 { - t.Errorf("expected pivot node %s, got %s", id2, *pid) - } -} - func TestStartStopNode(t *testing.T) { sim := New(noopServiceFuncMap) defer sim.Close() diff --git a/swarm/network/simulation/simulation.go b/swarm/network/simulation/simulation.go index 13c5b1c575..e18d19a67c 100644 --- a/swarm/network/simulation/simulation.go +++ b/swarm/network/simulation/simulation.go @@ -46,7 +46,6 @@ type Simulation struct { serviceNames []string cleanupFuncs []func() buckets map[enode.ID]*sync.Map - pivotNodeID *enode.ID shutdownWG sync.WaitGroup done chan struct{} mu sync.RWMutex From d5cad488be0069d768b358b2267cd5432b0f9a43 Mon Sep 17 00:00:00 2001 From: gary rong Date: Fri, 11 Jan 2019 19:49:12 +0800 Subject: [PATCH 14/14] core, eth: fix database version (#18429) * core, eth: fix database version * eth: polish error message --- core/blockchain.go | 2 +- core/rawdb/accessors_metadata.go | 22 +++++++++++++++------- eth/backend.go | 6 ++++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index c29063a73b..49aedf6693 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -65,7 +65,7 @@ const ( triesInMemory = 128 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. - BlockChainVersion = 3 + BlockChainVersion uint64 = 3 ) // CacheConfig contains the configuration values for the trie caching/pruning diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 3b6e6548d3..82e4bf0455 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -26,19 +26,27 @@ import ( ) // ReadDatabaseVersion retrieves the version number of the database. -func ReadDatabaseVersion(db DatabaseReader) int { - var version int +func ReadDatabaseVersion(db DatabaseReader) *uint64 { + var version uint64 enc, _ := db.Get(databaseVerisionKey) - rlp.DecodeBytes(enc, &version) + if len(enc) == 0 { + return nil + } + if err := rlp.DecodeBytes(enc, &version); err != nil { + return nil + } - return version + return &version } // WriteDatabaseVersion stores the version number of the database -func WriteDatabaseVersion(db DatabaseWriter, version int) { - enc, _ := rlp.EncodeToBytes(version) - if err := db.Put(databaseVerisionKey, enc); err != nil { +func WriteDatabaseVersion(db DatabaseWriter, version uint64) { + enc, err := rlp.EncodeToBytes(version) + if err != nil { + log.Crit("Failed to encode database version", "err", err) + } + if err = db.Put(databaseVerisionKey, enc); err != nil { log.Crit("Failed to store the database version", "err", err) } } diff --git a/eth/backend.go b/eth/backend.go index 354fc17d48..2a9d56c5c1 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -143,8 +143,10 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if !config.SkipBcVersionCheck { bcVersion := rawdb.ReadDatabaseVersion(chainDb) - if bcVersion != core.BlockChainVersion && bcVersion != 0 { - return nil, fmt.Errorf("Blockchain DB version mismatch (%d / %d).\n", bcVersion, core.BlockChainVersion) + if bcVersion != nil && *bcVersion > core.BlockChainVersion { + return nil, fmt.Errorf("database version is v%d, Geth %s only supports v%d", *bcVersion, params.VersionWithMeta, core.BlockChainVersion) + } else if bcVersion != nil && *bcVersion < core.BlockChainVersion { + log.Warn("Upgrade blockchain database version", "from", *bcVersion, "to", core.BlockChainVersion) } rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) }