diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 501cb16213..4dae586535 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -119,11 +119,22 @@ func unpack(t *Type, dst interface{}, src interface{}) error { dstVal = reflect.ValueOf(dst).Elem() srcVal = reflect.ValueOf(src) ) - - if t.T != TupleTy && !((t.T == SliceTy || t.T == ArrayTy) && t.Elem.T == TupleTy) { + tuple, typ := false, t + for { + if typ.T == SliceTy || typ.T == ArrayTy { + typ = typ.Elem + continue + } + tuple = typ.T == TupleTy + break + } + if !tuple { return set(dstVal, srcVal) } + // Dereferences interface or pointer wrapper + dstVal = indirectInterfaceOrPtr(dstVal) + switch t.T { case TupleTy: if dstVal.Kind() != reflect.Struct { @@ -191,7 +202,7 @@ func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues interfac argument := arguments.NonIndexed()[0] elem := reflect.ValueOf(v).Elem() - if elem.Kind() == reflect.Struct { + if elem.Kind() == reflect.Struct && argument.Type.T != TupleTy { fieldmap, err := mapArgNamesToStructFields([]string{argument.Name}, elem) if err != nil { return err diff --git a/accounts/abi/bind/auth.go b/accounts/abi/bind/auth.go index 5d5ad0034c..e51f0bd8ea 100644 --- a/accounts/abi/bind/auth.go +++ b/accounts/abi/bind/auth.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/accounts/external" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -43,7 +44,7 @@ func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) { return NewKeyedTransactor(key.PrivateKey), nil } -// NewKeystoreTransactor is a utility method to easily create a transaction signer from +// NewKeyStoreTransactor is a utility method to easily create a transaction signer from // an decrypted key from a keystore func NewKeyStoreTransactor(keystore *keystore.KeyStore, account accounts.Account) (*TransactOpts, error) { return &TransactOpts{ @@ -79,3 +80,17 @@ func NewKeyedTransactor(key *ecdsa.PrivateKey) *TransactOpts { }, } } + +// NewClefTransactor is a utility method to easily create a transaction signer +// with a clef backend. +func NewClefTransactor(clef *external.ExternalSigner, account accounts.Account) *TransactOpts { + return &TransactOpts{ + From: account.Address, + Signer: func(signer types.Signer, address common.Address, transaction *types.Transaction) (*types.Transaction, error) { + if address != account.Address { + return nil, errors.New("not authorized to sign this account") + } + return clef.SignTx(account, transaction, nil) // Clef enforces its own chain id + }, + } +} diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 4c9715b740..6106b380c7 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -22,6 +22,7 @@ package bind import ( "bytes" + "errors" "fmt" "go/format" "regexp" @@ -38,6 +39,7 @@ type Lang int const ( LangGo Lang = iota LangJava + LangObjC ) // Bind generates a Go wrapper around a contract ABI. This wrapper isn't meant @@ -62,11 +64,12 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] return r }, abis[i]) - // Extract the call and transact methods; events; and sort them alphabetically + // Extract the call and transact methods; events, struct definitions; and sort them alphabetically var ( calls = make(map[string]*tmplMethod) transacts = make(map[string]*tmplMethod) events = make(map[string]*tmplEvent) + structs = make(map[string]*tmplStruct) ) for _, original := range evmABI.Methods { // Normalize the method for capital cases and non-anonymous inputs/outputs @@ -79,6 +82,9 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] if input.Name == "" { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } + if _, exist := structs[input.Type.String()]; input.Type.T == abi.TupleTy && !exist { + bindStructType[lang](input.Type, structs) + } } normalized.Outputs = make([]abi.Argument, len(original.Outputs)) copy(normalized.Outputs, original.Outputs) @@ -86,6 +92,9 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] if output.Name != "" { normalized.Outputs[j].Name = capitalise(output.Name) } + if _, exist := structs[output.Type.String()]; output.Type.T == abi.TupleTy && !exist { + bindStructType[lang](output.Type, structs) + } } // Append the methods to the call or transact lists if original.Const { @@ -111,11 +120,20 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] if input.Name == "" { normalized.Inputs[j].Name = fmt.Sprintf("arg%d", j) } + if _, exist := structs[input.Type.String()]; input.Type.T == abi.TupleTy && !exist { + bindStructType[lang](input.Type, structs) + } } } // Append the event to the accumulator list events[original.Name] = &tmplEvent{Original: original, Normalized: normalized} } + + // There is no easy way to pass arbitrary java objects to the Go side. + if len(structs) > 0 && lang == LangJava { + return "", errors.New("java binding for tuple arguments is not supported yet") + } + contracts[types[i]] = &tmplContract{ Type: capitalise(types[i]), InputABI: strings.Replace(strippedABI, "\"", "\\\"", -1), @@ -124,6 +142,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] Calls: calls, Transacts: transacts, Events: events, + Structs: structs, } if len(fsigs) > i { contracts[types[i]].FuncSigs = fsigs[i] @@ -140,6 +159,8 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] "bindtype": bindType[lang], "bindtopictype": bindTopicType[lang], "namedtype": namedType[lang], + "formatmethod": formatMethod, + "formatevent": formatEvent, "capitalise": capitalise, "decapitalise": decapitalise, } @@ -161,7 +182,7 @@ func Bind(types []string, abis []string, bytecodes []string, fsigs []map[string] // bindType is a set of type binders that convert Solidity types to some supported // programming language types. -var bindType = map[Lang]func(kind abi.Type) string{ +var bindType = map[Lang]func(kind abi.Type, structs map[string]*tmplStruct) string{ LangGo: bindTypeGo, LangJava: bindTypeJava, } @@ -193,13 +214,14 @@ func bindBasicTypeGo(kind abi.Type) string { // bindTypeGo converts solidity types to Go ones. Since there is no clear mapping // from all Solidity types to Go ones (e.g. uint17), those that cannot be exactly // mapped will use an upscaled type (e.g. BigDecimal). -func bindTypeGo(kind abi.Type) string { - // todo(rjl493456442) tuple +func bindTypeGo(kind abi.Type, structs map[string]*tmplStruct) string { switch kind.T { + case abi.TupleTy: + return structs[kind.String()].Name case abi.ArrayTy: - return fmt.Sprintf("[%d]", kind.Size) + bindTypeGo(*kind.Elem) + return fmt.Sprintf("[%d]", kind.Size) + bindTypeGo(*kind.Elem, structs) case abi.SliceTy: - return "[]" + bindTypeGo(*kind.Elem) + return "[]" + bindTypeGo(*kind.Elem, structs) default: return bindBasicTypeGo(kind) } @@ -248,27 +270,33 @@ func bindBasicTypeJava(kind abi.Type) string { } } +// pluralizeJavaType explicitly converts multidimensional types to predefined +// type in go side. +func pluralizeJavaType(typ string) string { + switch typ { + case "boolean": + return "Bools" + case "String": + return "Strings" + case "Address": + return "Addresses" + case "byte[]": + return "Binaries" + case "BigInt": + return "BigInts" + } + return typ + "[]" +} + // bindTypeJava converts a Solidity type to a Java one. Since there is no clear mapping // from all Solidity types to Java ones (e.g. uint17), those that cannot be exactly // mapped will use an upscaled type (e.g. BigDecimal). -func bindTypeJava(kind abi.Type) string { +func bindTypeJava(kind abi.Type, structs map[string]*tmplStruct) string { switch kind.T { + case abi.TupleTy: + return structs[kind.String()].Name case abi.ArrayTy, abi.SliceTy: - // Explicitly convert multidimensional types to predefined type in go side. - inner := bindTypeJava(*kind.Elem) - switch inner { - case "boolean": - return "Bools" - case "String": - return "Strings" - case "Address": - return "Addresses" - case "byte[]": - return "Binaries" - case "BigInt": - return "BigInts" - } - return inner + "[]" + return pluralizeJavaType(bindTypeJava(*kind.Elem, structs)) default: return bindBasicTypeJava(kind) } @@ -276,15 +304,15 @@ func bindTypeJava(kind abi.Type) string { // bindTopicType is a set of type binders that convert Solidity types to some // supported programming language topic types. -var bindTopicType = map[Lang]func(kind abi.Type) string{ +var bindTopicType = map[Lang]func(kind abi.Type, structs map[string]*tmplStruct) string{ LangGo: bindTopicTypeGo, LangJava: bindTopicTypeJava, } // bindTypeGo converts a Solidity topic type to a Go one. It is almost the same // funcionality as for simple types, but dynamic types get converted to hashes. -func bindTopicTypeGo(kind abi.Type) string { - bound := bindTypeGo(kind) +func bindTopicTypeGo(kind abi.Type, structs map[string]*tmplStruct) string { + bound := bindTypeGo(kind, structs) if bound == "string" || bound == "[]byte" { bound = "common.Hash" } @@ -293,14 +321,77 @@ func bindTopicTypeGo(kind abi.Type) string { // bindTypeGo converts a Solidity topic type to a Java one. It is almost the same // funcionality as for simple types, but dynamic types get converted to hashes. -func bindTopicTypeJava(kind abi.Type) string { - bound := bindTypeJava(kind) +func bindTopicTypeJava(kind abi.Type, structs map[string]*tmplStruct) string { + bound := bindTypeJava(kind, structs) if bound == "String" || bound == "byte[]" { bound = "Hash" } return bound } +// bindStructType is a set of type binders that convert Solidity tuple types to some supported +// programming language struct definition. +var bindStructType = map[Lang]func(kind abi.Type, structs map[string]*tmplStruct) string{ + LangGo: bindStructTypeGo, + LangJava: bindStructTypeJava, +} + +// bindStructTypeGo converts a Solidity tuple type to a Go one and records the mapping +// in the given map. +// Notably, this function will resolve and record nested struct recursively. +func bindStructTypeGo(kind abi.Type, structs map[string]*tmplStruct) string { + switch kind.T { + case abi.TupleTy: + if s, exist := structs[kind.String()]; exist { + return s.Name + } + var fields []*tmplField + for i, elem := range kind.TupleElems { + field := bindStructTypeGo(*elem, structs) + fields = append(fields, &tmplField{Type: field, Name: capitalise(kind.TupleRawNames[i]), SolKind: *elem}) + } + name := fmt.Sprintf("Struct%d", len(structs)) + structs[kind.String()] = &tmplStruct{ + Name: name, + Fields: fields, + } + return name + case abi.ArrayTy: + return fmt.Sprintf("[%d]", kind.Size) + bindStructTypeGo(*kind.Elem, structs) + case abi.SliceTy: + return "[]" + bindStructTypeGo(*kind.Elem, structs) + default: + return bindBasicTypeGo(kind) + } +} + +// bindStructTypeJava converts a Solidity tuple type to a Java one and records the mapping +// in the given map. +// Notably, this function will resolve and record nested struct recursively. +func bindStructTypeJava(kind abi.Type, structs map[string]*tmplStruct) string { + switch kind.T { + case abi.TupleTy: + if s, exist := structs[kind.String()]; exist { + return s.Name + } + var fields []*tmplField + for i, elem := range kind.TupleElems { + field := bindStructTypeJava(*elem, structs) + fields = append(fields, &tmplField{Type: field, Name: decapitalise(kind.TupleRawNames[i]), SolKind: *elem}) + } + name := fmt.Sprintf("Class%d", len(structs)) + structs[kind.String()] = &tmplStruct{ + Name: name, + Fields: fields, + } + return name + case abi.ArrayTy, abi.SliceTy: + return pluralizeJavaType(bindStructTypeJava(*kind.Elem, structs)) + default: + return bindBasicTypeJava(kind) + } +} + // namedType is a set of functions that transform language specific types to // named versions that my be used inside method names. var namedType = map[Lang]func(string, abi.Type) string{ @@ -378,3 +469,63 @@ func structured(args abi.Arguments) bool { } return true } + +// resolveArgName converts a raw argument representation into a user friendly format. +func resolveArgName(arg abi.Argument, structs map[string]*tmplStruct) string { + var ( + prefix string + embedded string + typ = &arg.Type + ) +loop: + for { + switch typ.T { + case abi.SliceTy: + prefix += "[]" + case abi.ArrayTy: + prefix += fmt.Sprintf("[%d]", typ.Size) + default: + embedded = typ.String() + break loop + } + typ = typ.Elem + } + if s, exist := structs[embedded]; exist { + return prefix + s.Name + } else { + return arg.Type.String() + } +} + +// formatMethod transforms raw method representation into a user friendly one. +func formatMethod(method abi.Method, structs map[string]*tmplStruct) string { + inputs := make([]string, len(method.Inputs)) + for i, input := range method.Inputs { + inputs[i] = fmt.Sprintf("%v %v", resolveArgName(input, structs), input.Name) + } + outputs := make([]string, len(method.Outputs)) + for i, output := range method.Outputs { + outputs[i] = resolveArgName(output, structs) + if len(output.Name) > 0 { + outputs[i] += fmt.Sprintf(" %v", output.Name) + } + } + constant := "" + if method.Const { + constant = "constant " + } + return fmt.Sprintf("function %v(%v) %sreturns(%v)", method.Name, strings.Join(inputs, ", "), constant, strings.Join(outputs, ", ")) +} + +// formatEvent transforms raw event representation into a user friendly one. +func formatEvent(event abi.Event, structs map[string]*tmplStruct) string { + inputs := make([]string, len(event.Inputs)) + for i, input := range event.Inputs { + if input.Indexed { + inputs[i] = fmt.Sprintf("%v indexed %v", resolveArgName(input, structs), input.Name) + } else { + inputs[i] = fmt.Sprintf("%v %v", resolveArgName(input, structs), input.Name) + } + } + return fmt.Sprintf("event %v(%v)", event.Name, strings.Join(inputs, ", ")) +} diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 35e9ce0e04..6dca96ae34 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1014,6 +1014,137 @@ var bindTests = []struct { "test(function)": "d7a5aba2", }, }, + }, { + `Tuple`, + ` + pragma solidity >=0.4.19 <0.6.0; + pragma experimental ABIEncoderV2; + + contract Tuple { + struct S { uint a; uint[] b; T[] c; } + struct T { uint x; uint y; } + event TupleEvent(S a, T[2][] b, T[][2] c, S[] d, uint[] e); + + function func1(S memory a, T[2][] memory b, T[][2] memory c, S[] memory d, uint[] memory e) public pure returns (S memory, T[2][] memory, T[][2] memory, S[] memory, uint[] memory) { + return (a, b, c, d, e); + } + function func2(S memory a, T[2][] memory b, T[][2] memory c, S[] memory d, uint[] memory e) public { + emit TupleEvent(a, b, c, d, e); + } + } + + `, + `608060405234801561001057600080fd5b50610eb2806100206000396000f3fe608060405234801561001057600080fd5b50600436106100365760003560e01c8063443c79b41461003b578063d0062cdd1461006f575b600080fd5b61005560048036036100509190810190610624565b61008b565b604051610066959493929190610b28565b60405180910390f35b61008960048036036100849190810190610624565b6100bc565b005b610093610102565b606061009d610123565b6060808989898989945094509450945094509550955095509550959050565b7f18d6e66efa53739ca6d13626f35ebc700b31cced3eddb50c70bbe9c082c6cd0085858585856040516100f3959493929190610b28565b60405180910390a15050505050565b60405180606001604052806000815260200160608152602001606081525090565b60405180604001604052806002905b60608152602001906001900390816101325790505090565b600082601f83011261015b57600080fd5b813561016e61016982610bcb565b610b9e565b9150818183526020840193506020810190508385608084028201111561019357600080fd5b60005b838110156101c357816101a988826102a6565b845260208401935060808301925050600181019050610196565b5050505092915050565b600082601f8301126101de57600080fd5b60026101f16101ec82610bf3565b610b9e565b9150818360005b83811015610228578135860161020e888261031a565b8452602084019350602083019250506001810190506101f8565b5050505092915050565b600082601f83011261024357600080fd5b813561025661025182610c15565b610b9e565b9150818183526020840193506020810190508360005b8381101561029c578135860161028288826104a3565b84526020840193506020830192505060018101905061026c565b5050505092915050565b600082601f8301126102b757600080fd5b60026102ca6102c582610c3d565b610b9e565b915081838560408402820111156102e057600080fd5b60005b8381101561031057816102f688826105c3565b8452602084019350604083019250506001810190506102e3565b5050505092915050565b600082601f83011261032b57600080fd5b813561033e61033982610c5f565b610b9e565b9150818183526020840193506020810190508385604084028201111561036357600080fd5b60005b83811015610393578161037988826105c3565b845260208401935060408301925050600181019050610366565b5050505092915050565b600082601f8301126103ae57600080fd5b81356103c16103bc82610c87565b610b9e565b915081818352602084019350602081019050838560208402820111156103e657600080fd5b60005b8381101561041657816103fc888261060f565b8452602084019350602083019250506001810190506103e9565b5050505092915050565b600082601f83011261043157600080fd5b813561044461043f82610caf565b610b9e565b9150818183526020840193506020810190508385602084028201111561046957600080fd5b60005b83811015610499578161047f888261060f565b84526020840193506020830192505060018101905061046c565b5050505092915050565b6000606082840312156104b557600080fd5b6104bf6060610b9e565b905060006104cf8482850161060f565b600083015250602082013567ffffffffffffffff8111156104ef57600080fd5b6104fb8482850161039d565b602083015250604082013567ffffffffffffffff81111561051b57600080fd5b6105278482850161031a565b60408301525092915050565b60006060828403121561054557600080fd5b61054f6060610b9e565b9050600061055f8482850161060f565b600083015250602082013567ffffffffffffffff81111561057f57600080fd5b61058b8482850161039d565b602083015250604082013567ffffffffffffffff8111156105ab57600080fd5b6105b78482850161031a565b60408301525092915050565b6000604082840312156105d557600080fd5b6105df6040610b9e565b905060006105ef8482850161060f565b60008301525060206106038482850161060f565b60208301525092915050565b60008135905061061e81610e58565b92915050565b600080600080600060a0868803121561063c57600080fd5b600086013567ffffffffffffffff81111561065657600080fd5b61066288828901610533565b955050602086013567ffffffffffffffff81111561067f57600080fd5b61068b8882890161014a565b945050604086013567ffffffffffffffff8111156106a857600080fd5b6106b4888289016101cd565b935050606086013567ffffffffffffffff8111156106d157600080fd5b6106dd88828901610232565b925050608086013567ffffffffffffffff8111156106fa57600080fd5b61070688828901610420565b9150509295509295909350565b600061071f83836108cb565b60808301905092915050565b60006107378383610922565b905092915050565b600061074b8383610a93565b905092915050565b600061075f8383610aea565b60408301905092915050565b60006107778383610b19565b60208301905092915050565b600061078e82610d3b565b6107988185610de3565b93506107a383610cd7565b8060005b838110156107d45781516107bb8882610713565b97506107c683610d88565b9250506001810190506107a7565b5085935050505092915050565b60006107ec82610d46565b6107f68185610df4565b93508360208202850161080885610ce7565b8060005b858110156108445784840389528151610825858261072b565b945061083083610d95565b925060208a0199505060018101905061080c565b50829750879550505050505092915050565b600061086182610d51565b61086b8185610dff565b93508360208202850161087d85610cf1565b8060005b858110156108b9578484038952815161089a858261073f565b94506108a583610da2565b925060208a01995050600181019050610881565b50829750879550505050505092915050565b6108d481610d5c565b6108de8184610e10565b92506108e982610d01565b8060005b8381101561091a5781516109018782610753565b965061090c83610daf565b9250506001810190506108ed565b505050505050565b600061092d82610d67565b6109378185610e1b565b935061094283610d0b565b8060005b8381101561097357815161095a8882610753565b975061096583610dbc565b925050600181019050610946565b5085935050505092915050565b600061098b82610d7d565b6109958185610e3d565b93506109a083610d2b565b8060005b838110156109d15781516109b8888261076b565b97506109c383610dd6565b9250506001810190506109a4565b5085935050505092915050565b60006109e982610d72565b6109f38185610e2c565b93506109fe83610d1b565b8060005b83811015610a2f578151610a16888261076b565b9750610a2183610dc9565b925050600181019050610a02565b5085935050505092915050565b6000606083016000830151610a546000860182610b19565b5060208301518482036020860152610a6c82826109de565b91505060408301518482036040860152610a868282610922565b9150508091505092915050565b6000606083016000830151610aab6000860182610b19565b5060208301518482036020860152610ac382826109de565b91505060408301518482036040860152610add8282610922565b9150508091505092915050565b604082016000820151610b006000850182610b19565b506020820151610b136020850182610b19565b50505050565b610b2281610e4e565b82525050565b600060a0820190508181036000830152610b428188610a3c565b90508181036020830152610b568187610783565b90508181036040830152610b6a81866107e1565b90508181036060830152610b7e8185610856565b90508181036080830152610b928184610980565b90509695505050505050565b6000604051905081810181811067ffffffffffffffff82111715610bc157600080fd5b8060405250919050565b600067ffffffffffffffff821115610be257600080fd5b602082029050602081019050919050565b600067ffffffffffffffff821115610c0a57600080fd5b602082029050919050565b600067ffffffffffffffff821115610c2c57600080fd5b602082029050602081019050919050565b600067ffffffffffffffff821115610c5457600080fd5b602082029050919050565b600067ffffffffffffffff821115610c7657600080fd5b602082029050602081019050919050565b600067ffffffffffffffff821115610c9e57600080fd5b602082029050602081019050919050565b600067ffffffffffffffff821115610cc657600080fd5b602082029050602081019050919050565b6000819050602082019050919050565b6000819050919050565b6000819050602082019050919050565b6000819050919050565b6000819050602082019050919050565b6000819050602082019050919050565b6000819050602082019050919050565b600081519050919050565b600060029050919050565b600081519050919050565b600060029050919050565b600081519050919050565b600081519050919050565b600081519050919050565b6000602082019050919050565b6000602082019050919050565b6000602082019050919050565b6000602082019050919050565b6000602082019050919050565b6000602082019050919050565b6000602082019050919050565b600082825260208201905092915050565b600081905092915050565b600082825260208201905092915050565b600081905092915050565b600082825260208201905092915050565b600082825260208201905092915050565b600082825260208201905092915050565b6000819050919050565b610e6181610e4e565b8114610e6c57600080fd5b5056fea365627a7a72305820405a6336d8c302cee779de6788527018e5a2393892328fbf12b96065df2de00a6c6578706572696d656e74616cf564736f6c634300050a0040`, + ` + [{"constant":true,"inputs":[{"components":[{"name":"a","type":"uint256"},{"name":"b","type":"uint256[]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"c","type":"tuple[]"}],"name":"a","type":"tuple"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"b","type":"tuple[2][]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"c","type":"tuple[][2]"},{"components":[{"name":"a","type":"uint256"},{"name":"b","type":"uint256[]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"c","type":"tuple[]"}],"name":"d","type":"tuple[]"},{"name":"e","type":"uint256[]"}],"name":"func1","outputs":[{"components":[{"name":"a","type":"uint256"},{"name":"b","type":"uint256[]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"c","type":"tuple[]"}],"name":"","type":"tuple"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"","type":"tuple[2][]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"","type":"tuple[][2]"},{"components":[{"name":"a","type":"uint256"},{"name":"b","type":"uint256[]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"c","type":"tuple[]"}],"name":"","type":"tuple[]"},{"name":"","type":"uint256[]"}],"payable":false,"stateMutability":"pure","type":"function"},{"constant":false,"inputs":[{"components":[{"name":"a","type":"uint256"},{"name":"b","type":"uint256[]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"c","type":"tuple[]"}],"name":"a","type":"tuple"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"b","type":"tuple[2][]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"c","type":"tuple[][2]"},{"components":[{"name":"a","type":"uint256"},{"name":"b","type":"uint256[]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"c","type":"tuple[]"}],"name":"d","type":"tuple[]"},{"name":"e","type":"uint256[]"}],"name":"func2","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"anonymous":false,"inputs":[{"components":[{"name":"a","type":"uint256"},{"name":"b","type":"uint256[]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"c","type":"tuple[]"}],"indexed":false,"name":"a","type":"tuple"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"indexed":false,"name":"b","type":"tuple[2][]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"indexed":false,"name":"c","type":"tuple[][2]"},{"components":[{"name":"a","type":"uint256"},{"name":"b","type":"uint256[]"},{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"c","type":"tuple[]"}],"indexed":false,"name":"d","type":"tuple[]"},{"indexed":false,"name":"e","type":"uint256[]"}],"name":"TupleEvent","type":"event"}] + `, + ` + "math/big" + "reflect" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, + + ` + key, _ := crypto.GenerateKey() + auth := bind.NewKeyedTransactor(key) + backend := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) + + _, _, contract, err := DeployTuple(auth, backend) + if err != nil { + t.Fatalf("deploy contract failed %v", err) + } + backend.Commit() + + check := func(a, b interface{}, errMsg string) { + if !reflect.DeepEqual(a, b) { + t.Fatal(errMsg) + } + } + + a := Struct1{ + A: big.NewInt(1), + B: []*big.Int{big.NewInt(2), big.NewInt(3)}, + C: []Struct0{ + { + X: big.NewInt(4), + Y: big.NewInt(5), + }, + { + X: big.NewInt(6), + Y: big.NewInt(7), + }, + }, + } + + b := [][2]Struct0{ + { + { + X: big.NewInt(8), + Y: big.NewInt(9), + }, + { + X: big.NewInt(10), + Y: big.NewInt(11), + }, + }, + } + + c := [2][]Struct0{ + { + { + X: big.NewInt(12), + Y: big.NewInt(13), + }, + { + X: big.NewInt(14), + Y: big.NewInt(15), + }, + }, + { + { + X: big.NewInt(16), + Y: big.NewInt(17), + }, + }, + } + + d := []Struct1{a} + + e := []*big.Int{big.NewInt(18), big.NewInt(19)} + ret1, ret2, ret3, ret4, ret5, err := contract.Func1(nil, a, b, c, d, e) + if err != nil { + t.Fatalf("invoke contract failed, err %v", err) + } + check(ret1, a, "ret1 mismatch") + check(ret2, b, "ret2 mismatch") + check(ret3, c, "ret3 mismatch") + check(ret4, d, "ret4 mismatch") + check(ret5, e, "ret5 mismatch") + + _, err = contract.Func2(auth, a, b, c, d, e) + if err != nil { + t.Fatalf("invoke contract failed, err %v", err) + } + backend.Commit() + + iter, err := contract.FilterTupleEvent(nil) + if err != nil { + t.Fatalf("failed to create event filter, err %v", err) + } + defer iter.Close() + + iter.Next() + check(iter.Event.A, a, "field1 mismatch") + check(iter.Event.B, b, "field2 mismatch") + check(iter.Event.C, c, "field3 mismatch") + check(iter.Event.D, d, "field4 mismatch") + check(iter.Event.E, e, "field5 mismatch") + `, + nil, }, } @@ -1133,6 +1264,7 @@ import org.ethereum.geth.*; import java.util.*; + public class Test { // ABI is the input ABI used to generate the binding from. public final static String ABI = "[{\"constant\":false,\"inputs\":[{\"name\":\"u16\",\"type\":\"uint16\"}],\"name\":\"setUint16\",\"outputs\":[{\"name\":\"\",\"type\":\"uint16\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"b_a\",\"type\":\"bool[2]\"}],\"name\":\"setBoolArray\",\"outputs\":[{\"name\":\"\",\"type\":\"bool[2]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"a_a\",\"type\":\"address[2]\"}],\"name\":\"setAddressArray\",\"outputs\":[{\"name\":\"\",\"type\":\"address[2]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"bs_l\",\"type\":\"bytes[]\"}],\"name\":\"setBytesList\",\"outputs\":[{\"name\":\"\",\"type\":\"bytes[]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"u8\",\"type\":\"uint8\"}],\"name\":\"setUint8\",\"outputs\":[{\"name\":\"\",\"type\":\"uint8\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"u32\",\"type\":\"uint32\"}],\"name\":\"setUint32\",\"outputs\":[{\"name\":\"\",\"type\":\"uint32\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"b\",\"type\":\"bool\"}],\"name\":\"setBool\",\"outputs\":[{\"name\":\"\",\"type\":\"bool\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"i256_l\",\"type\":\"int256[]\"}],\"name\":\"setInt256List\",\"outputs\":[{\"name\":\"\",\"type\":\"int256[]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"u256_a\",\"type\":\"uint256[2]\"}],\"name\":\"setUint256Array\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256[2]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"b_l\",\"type\":\"bool[]\"}],\"name\":\"setBoolList\",\"outputs\":[{\"name\":\"\",\"type\":\"bool[]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"bs_a\",\"type\":\"bytes[2]\"}],\"name\":\"setBytesArray\",\"outputs\":[{\"name\":\"\",\"type\":\"bytes[2]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"a_l\",\"type\":\"address[]\"}],\"name\":\"setAddressList\",\"outputs\":[{\"name\":\"\",\"type\":\"address[]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"i256_a\",\"type\":\"int256[2]\"}],\"name\":\"setInt256Array\",\"outputs\":[{\"name\":\"\",\"type\":\"int256[2]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"s_a\",\"type\":\"string[2]\"}],\"name\":\"setStringArray\",\"outputs\":[{\"name\":\"\",\"type\":\"string[2]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"s\",\"type\":\"string\"}],\"name\":\"setString\",\"outputs\":[{\"name\":\"\",\"type\":\"string\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"u64\",\"type\":\"uint64\"}],\"name\":\"setUint64\",\"outputs\":[{\"name\":\"\",\"type\":\"uint64\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"i16\",\"type\":\"int16\"}],\"name\":\"setInt16\",\"outputs\":[{\"name\":\"\",\"type\":\"int16\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"i8\",\"type\":\"int8\"}],\"name\":\"setInt8\",\"outputs\":[{\"name\":\"\",\"type\":\"int8\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"u256_l\",\"type\":\"uint256[]\"}],\"name\":\"setUint256List\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256[]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"i256\",\"type\":\"int256\"}],\"name\":\"setInt256\",\"outputs\":[{\"name\":\"\",\"type\":\"int256\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"i32\",\"type\":\"int32\"}],\"name\":\"setInt32\",\"outputs\":[{\"name\":\"\",\"type\":\"int32\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"b32\",\"type\":\"bytes32\"}],\"name\":\"setBytes32\",\"outputs\":[{\"name\":\"\",\"type\":\"bytes32\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"s_l\",\"type\":\"string[]\"}],\"name\":\"setStringList\",\"outputs\":[{\"name\":\"\",\"type\":\"string[]\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"u256\",\"type\":\"uint256\"}],\"name\":\"setUint256\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"bs\",\"type\":\"bytes\"}],\"name\":\"setBytes\",\"outputs\":[{\"name\":\"\",\"type\":\"bytes\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"a\",\"type\":\"address\"}],\"name\":\"setAddress\",\"outputs\":[{\"name\":\"\",\"type\":\"address\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"i64\",\"type\":\"int64\"}],\"name\":\"setInt64\",\"outputs\":[{\"name\":\"\",\"type\":\"int64\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"b1\",\"type\":\"bytes1\"}],\"name\":\"setBytes1\",\"outputs\":[{\"name\":\"\",\"type\":\"bytes1\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"}]"; diff --git a/accounts/abi/bind/template.go b/accounts/abi/bind/template.go index 127bd6d394..32e7eb1e8c 100644 --- a/accounts/abi/bind/template.go +++ b/accounts/abi/bind/template.go @@ -34,6 +34,7 @@ type tmplContract struct { Calls map[string]*tmplMethod // Contract calls that only read state data Transacts map[string]*tmplMethod // Contract calls that write state data Events map[string]*tmplEvent // Contract events accessors + Structs map[string]*tmplStruct // Contract struct type definitions } // tmplMethod is a wrapper around an abi.Method that contains a few preprocessed @@ -50,6 +51,21 @@ type tmplEvent struct { Normalized abi.Event // Normalized version of the parsed fields } +// tmplField is a wrapper around a struct field with binding language +// struct type definition and relative filed name. +type tmplField struct { + Type string // Field type representation depends on target binding language + Name string // Field name converted from the raw user-defined field name + SolKind abi.Type // Raw abi type information +} + +// tmplStruct is a wrapper around an abi.tuple contains a auto-generated +// struct name. +type tmplStruct struct { + Name string // Auto-generated struct name(We can't obtain the raw struct name through abi) + Fields []*tmplField // Struct fields definition depends on the binding language. +} + // tmplSource is language to template mapping containing all the supported // programming languages the package can generate to. var tmplSource = map[Lang]string{ @@ -90,6 +106,7 @@ var ( ) {{range $contract := .Contracts}} + {{$structs := $contract.Structs}} // {{.Type}}ABI is the input ABI used to generate the binding from. const {{.Type}}ABI = "{{.InputABI}}" @@ -107,7 +124,7 @@ var ( const {{.Type}}Bin = ` + "`" + `{{.InputBin}}` + "`" + ` // Deploy{{.Type}} deploys a new Ethereum contract, binding an instance of {{.Type}} to it. - func Deploy{{.Type}}(auth *bind.TransactOpts, backend bind.ContractBackend {{range .Constructor.Inputs}}, {{.Name}} {{bindtype .Type}}{{end}}) (common.Address, *types.Transaction, *{{.Type}}, error) { + func Deploy{{.Type}}(auth *bind.TransactOpts, backend bind.ContractBackend {{range .Constructor.Inputs}}, {{.Name}} {{bindtype .Type $structs}}{{end}}) (common.Address, *types.Transaction, *{{.Type}}, error) { parsed, err := abi.JSON(strings.NewReader({{.Type}}ABI)) if err != nil { return common.Address{}, nil, nil, err @@ -124,7 +141,7 @@ var ( type {{.Type}} struct { {{.Type}}Caller // Read-only binding to the contract {{.Type}}Transactor // Write-only binding to the contract - {{.Type}}Filterer // Log filterer for contract events + {{.Type}}Filterer // Log filterer for contract events } // {{.Type}}Caller is an auto generated read-only Go binding around an Ethereum contract. @@ -262,16 +279,24 @@ var ( return _{{$contract.Type}}.Contract.contract.Transact(opts, method, params...) } + {{range .Structs}} + // {{.Name}} is an auto generated low-level Go binding around an user-defined struct. + type {{.Name}} struct { + {{range $field := .Fields}} + {{$field.Name}} {{$field.Type}}{{end}} + } + {{end}} + {{range .Calls}} // {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.Id}}. // - // Solidity: {{.Original.String}} - func (_{{$contract.Type}} *{{$contract.Type}}Caller) {{.Normalized.Name}}(opts *bind.CallOpts {{range .Normalized.Inputs}}, {{.Name}} {{bindtype .Type}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type}};{{end}} },{{else}}{{range .Normalized.Outputs}}{{bindtype .Type}},{{end}}{{end}} error) { + // Solidity: {{formatmethod .Original $structs}} + func (_{{$contract.Type}} *{{$contract.Type}}Caller) {{.Normalized.Name}}(opts *bind.CallOpts {{range .Normalized.Inputs}}, {{.Name}} {{bindtype .Type $structs}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type $structs}};{{end}} },{{else}}{{range .Normalized.Outputs}}{{bindtype .Type $structs}},{{end}}{{end}} error) { {{if .Structured}}ret := new(struct{ - {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type}} + {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type $structs}} {{end}} }){{else}}var ( - {{range $i, $_ := .Normalized.Outputs}}ret{{$i}} = new({{bindtype .Type}}) + {{range $i, $_ := .Normalized.Outputs}}ret{{$i}} = new({{bindtype .Type $structs}}) {{end}} ){{end}} out := {{if .Structured}}ret{{else}}{{if eq (len .Normalized.Outputs) 1}}ret0{{else}}&[]interface{}{ @@ -284,15 +309,15 @@ var ( // {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.Id}}. // - // Solidity: {{.Original.String}} - func (_{{$contract.Type}} *{{$contract.Type}}Session) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type}};{{end}} }, {{else}} {{range .Normalized.Outputs}}{{bindtype .Type}},{{end}} {{end}} error) { + // Solidity: {{formatmethod .Original $structs}} + func (_{{$contract.Type}} *{{$contract.Type}}Session) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type $structs}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type $structs}};{{end}} }, {{else}} {{range .Normalized.Outputs}}{{bindtype .Type $structs}},{{end}} {{end}} error) { return _{{$contract.Type}}.Contract.{{.Normalized.Name}}(&_{{$contract.Type}}.CallOpts {{range .Normalized.Inputs}}, {{.Name}}{{end}}) } // {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.Id}}. // - // Solidity: {{.Original.String}} - func (_{{$contract.Type}} *{{$contract.Type}}CallerSession) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type}};{{end}} }, {{else}} {{range .Normalized.Outputs}}{{bindtype .Type}},{{end}} {{end}} error) { + // Solidity: {{formatmethod .Original $structs}} + func (_{{$contract.Type}} *{{$contract.Type}}CallerSession) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type $structs}} {{end}}) ({{if .Structured}}struct{ {{range .Normalized.Outputs}}{{.Name}} {{bindtype .Type $structs}};{{end}} }, {{else}} {{range .Normalized.Outputs}}{{bindtype .Type $structs}},{{end}} {{end}} error) { return _{{$contract.Type}}.Contract.{{.Normalized.Name}}(&_{{$contract.Type}}.CallOpts {{range .Normalized.Inputs}}, {{.Name}}{{end}}) } {{end}} @@ -300,22 +325,22 @@ var ( {{range .Transacts}} // {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.Id}}. // - // Solidity: {{.Original.String}} - func (_{{$contract.Type}} *{{$contract.Type}}Transactor) {{.Normalized.Name}}(opts *bind.TransactOpts {{range .Normalized.Inputs}}, {{.Name}} {{bindtype .Type}} {{end}}) (*types.Transaction, error) { + // Solidity: {{formatmethod .Original $structs}} + func (_{{$contract.Type}} *{{$contract.Type}}Transactor) {{.Normalized.Name}}(opts *bind.TransactOpts {{range .Normalized.Inputs}}, {{.Name}} {{bindtype .Type $structs}} {{end}}) (*types.Transaction, error) { return _{{$contract.Type}}.contract.Transact(opts, "{{.Original.Name}}" {{range .Normalized.Inputs}}, {{.Name}}{{end}}) } // {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.Id}}. // - // Solidity: {{.Original.String}} - func (_{{$contract.Type}} *{{$contract.Type}}Session) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type}} {{end}}) (*types.Transaction, error) { + // Solidity: {{formatmethod .Original $structs}} + func (_{{$contract.Type}} *{{$contract.Type}}Session) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type $structs}} {{end}}) (*types.Transaction, error) { return _{{$contract.Type}}.Contract.{{.Normalized.Name}}(&_{{$contract.Type}}.TransactOpts {{range $i, $_ := .Normalized.Inputs}}, {{.Name}}{{end}}) } // {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.Id}}. // - // Solidity: {{.Original.String}} - func (_{{$contract.Type}} *{{$contract.Type}}TransactorSession) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type}} {{end}}) (*types.Transaction, error) { + // Solidity: {{formatmethod .Original $structs}} + func (_{{$contract.Type}} *{{$contract.Type}}TransactorSession) {{.Normalized.Name}}({{range $i, $_ := .Normalized.Inputs}}{{if ne $i 0}},{{end}} {{.Name}} {{bindtype .Type $structs}} {{end}}) (*types.Transaction, error) { return _{{$contract.Type}}.Contract.{{.Normalized.Name}}(&_{{$contract.Type}}.TransactOpts {{range $i, $_ := .Normalized.Inputs}}, {{.Name}}{{end}}) } {{end}} @@ -387,14 +412,14 @@ var ( // {{$contract.Type}}{{.Normalized.Name}} represents a {{.Normalized.Name}} event raised by the {{$contract.Type}} contract. type {{$contract.Type}}{{.Normalized.Name}} struct { {{range .Normalized.Inputs}} - {{capitalise .Name}} {{if .Indexed}}{{bindtopictype .Type}}{{else}}{{bindtype .Type}}{{end}}; {{end}} + {{capitalise .Name}} {{if .Indexed}}{{bindtopictype .Type $structs}}{{else}}{{bindtype .Type $structs}}{{end}}; {{end}} Raw types.Log // Blockchain specific contextual infos } // Filter{{.Normalized.Name}} is a free log retrieval operation binding the contract event 0x{{printf "%x" .Original.Id}}. // - // Solidity: {{.Original.String}} - func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Filter{{.Normalized.Name}}(opts *bind.FilterOpts{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type}}{{end}}{{end}}) (*{{$contract.Type}}{{.Normalized.Name}}Iterator, error) { + // Solidity: {{formatevent .Original $structs}} + func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Filter{{.Normalized.Name}}(opts *bind.FilterOpts{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type $structs}}{{end}}{{end}}) (*{{$contract.Type}}{{.Normalized.Name}}Iterator, error) { {{range .Normalized.Inputs}} {{if .Indexed}}var {{.Name}}Rule []interface{} for _, {{.Name}}Item := range {{.Name}} { @@ -410,8 +435,8 @@ var ( // Watch{{.Normalized.Name}} is a free log subscription operation binding the contract event 0x{{printf "%x" .Original.Id}}. // - // Solidity: {{.Original.String}} - func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Watch{{.Normalized.Name}}(opts *bind.WatchOpts, sink chan<- *{{$contract.Type}}{{.Normalized.Name}}{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type}}{{end}}{{end}}) (event.Subscription, error) { + // Solidity: {{formatevent .Original $structs}} + func (_{{$contract.Type}} *{{$contract.Type}}Filterer) Watch{{.Normalized.Name}}(opts *bind.WatchOpts, sink chan<- *{{$contract.Type}}{{.Normalized.Name}}{{range .Normalized.Inputs}}{{if .Indexed}}, {{.Name}} []{{bindtype .Type $structs}}{{end}}{{end}}) (event.Subscription, error) { {{range .Normalized.Inputs}} {{if .Indexed}}var {{.Name}}Rule []interface{} for _, {{.Name}}Item := range {{.Name}} { @@ -477,6 +502,7 @@ import org.ethereum.geth.*; import java.util.*; {{range $contract := .Contracts}} +{{$structs := $contract.Structs}} public class {{.Type}} { // ABI is the input ABI used to generate the binding from. public final static String ABI = "{{.InputABI}}"; @@ -496,9 +522,9 @@ public class {{.Type}} { public final static String BYTECODE = "0x{{.InputBin}}"; // deploy deploys a new Ethereum contract, binding an instance of {{.Type}} to it. - public static {{.Type}} deploy(TransactOpts auth, EthereumClient client{{range .Constructor.Inputs}}, {{bindtype .Type}} {{.Name}}{{end}}) throws Exception { + public static {{.Type}} deploy(TransactOpts auth, EthereumClient client{{range .Constructor.Inputs}}, {{bindtype .Type $structs}} {{.Name}}{{end}}) throws Exception { Interfaces args = Geth.newInterfaces({{(len .Constructor.Inputs)}}); - {{range $index, $element := .Constructor.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type) .Type}}({{.Name}});args.set({{$index}},arg{{$index}}); + {{range $index, $element := .Constructor.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type $structs) .Type}}({{.Name}});args.set({{$index}},arg{{$index}}); {{end}} return new {{.Type}}(Geth.deployContract(auth, ABI, Geth.decodeFromHex(BYTECODE), client, args)); } @@ -529,7 +555,7 @@ public class {{.Type}} { {{if gt (len .Normalized.Outputs) 1}} // {{capitalise .Normalized.Name}}Results is the output of a call to {{.Normalized.Name}}. public class {{capitalise .Normalized.Name}}Results { - {{range $index, $item := .Normalized.Outputs}}public {{bindtype .Type}} {{if ne .Name ""}}{{.Name}}{{else}}Return{{$index}}{{end}}; + {{range $index, $item := .Normalized.Outputs}}public {{bindtype .Type $structs}} {{if ne .Name ""}}{{.Name}}{{else}}Return{{$index}}{{end}}; {{end}} } {{end}} @@ -537,13 +563,13 @@ public class {{.Type}} { // {{.Normalized.Name}} is a free data retrieval call binding the contract method 0x{{printf "%x" .Original.Id}}. // // Solidity: {{.Original.String}} - public {{if gt (len .Normalized.Outputs) 1}}{{capitalise .Normalized.Name}}Results{{else}}{{range .Normalized.Outputs}}{{bindtype .Type}}{{end}}{{end}} {{.Normalized.Name}}(CallOpts opts{{range .Normalized.Inputs}}, {{bindtype .Type}} {{.Name}}{{end}}) throws Exception { + public {{if gt (len .Normalized.Outputs) 1}}{{capitalise .Normalized.Name}}Results{{else}}{{range .Normalized.Outputs}}{{bindtype .Type $structs}}{{end}}{{end}} {{.Normalized.Name}}(CallOpts opts{{range .Normalized.Inputs}}, {{bindtype .Type $structs}} {{.Name}}{{end}}) throws Exception { Interfaces args = Geth.newInterfaces({{(len .Normalized.Inputs)}}); - {{range $index, $item := .Normalized.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type) .Type}}({{.Name}});args.set({{$index}},arg{{$index}}); + {{range $index, $item := .Normalized.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type $structs) .Type}}({{.Name}});args.set({{$index}},arg{{$index}}); {{end}} Interfaces results = Geth.newInterfaces({{(len .Normalized.Outputs)}}); - {{range $index, $item := .Normalized.Outputs}}Interface result{{$index}} = Geth.newInterface(); result{{$index}}.setDefault{{namedtype (bindtype .Type) .Type}}(); results.set({{$index}}, result{{$index}}); + {{range $index, $item := .Normalized.Outputs}}Interface result{{$index}} = Geth.newInterface(); result{{$index}}.setDefault{{namedtype (bindtype .Type $structs) .Type}}(); results.set({{$index}}, result{{$index}}); {{end}} if (opts == null) { @@ -552,10 +578,10 @@ public class {{.Type}} { this.Contract.call(opts, results, "{{.Original.Name}}", args); {{if gt (len .Normalized.Outputs) 1}} {{capitalise .Normalized.Name}}Results result = new {{capitalise .Normalized.Name}}Results(); - {{range $index, $item := .Normalized.Outputs}}result.{{if ne .Name ""}}{{.Name}}{{else}}Return{{$index}}{{end}} = results.get({{$index}}).get{{namedtype (bindtype .Type) .Type}}(); + {{range $index, $item := .Normalized.Outputs}}result.{{if ne .Name ""}}{{.Name}}{{else}}Return{{$index}}{{end}} = results.get({{$index}}).get{{namedtype (bindtype .Type $structs) .Type}}(); {{end}} return result; - {{else}}{{range .Normalized.Outputs}}return results.get(0).get{{namedtype (bindtype .Type) .Type}}();{{end}} + {{else}}{{range .Normalized.Outputs}}return results.get(0).get{{namedtype (bindtype .Type $structs) .Type}}();{{end}} {{end}} } {{end}} @@ -564,9 +590,9 @@ public class {{.Type}} { // {{.Normalized.Name}} is a paid mutator transaction binding the contract method 0x{{printf "%x" .Original.Id}}. // // Solidity: {{.Original.String}} - public Transaction {{.Normalized.Name}}(TransactOpts opts{{range .Normalized.Inputs}}, {{bindtype .Type}} {{.Name}}{{end}}) throws Exception { + public Transaction {{.Normalized.Name}}(TransactOpts opts{{range .Normalized.Inputs}}, {{bindtype .Type $structs}} {{.Name}}{{end}}) throws Exception { Interfaces args = Geth.newInterfaces({{(len .Normalized.Inputs)}}); - {{range $index, $item := .Normalized.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type) .Type}}({{.Name}});args.set({{$index}},arg{{$index}}); + {{range $index, $item := .Normalized.Inputs}}Interface arg{{$index}} = Geth.newInterface();arg{{$index}}.set{{namedtype (bindtype .Type $structs) .Type}}({{.Name}});args.set({{$index}},arg{{$index}}); {{end}} return this.Contract.transact(opts, "{{.Original.Name}}" , args); } diff --git a/accounts/abi/method_test.go b/accounts/abi/method_test.go index a98f1cd31f..92c360f0d2 100644 --- a/accounts/abi/method_test.go +++ b/accounts/abi/method_test.go @@ -23,9 +23,13 @@ import ( const methoddata = ` [ - { "type" : "function", "name" : "balance", "constant" : true }, - { "type" : "function", "name" : "send", "constant" : false, "inputs" : [ { "name" : "amount", "type" : "uint256" } ] }, - { "type" : "function", "name" : "transfer", "constant" : false, "inputs" : [ { "name" : "from", "type" : "address" }, { "name" : "to", "type" : "address" }, { "name" : "value", "type" : "uint256" } ], "outputs" : [ { "name" : "success", "type" : "bool" } ] } + {"type": "function", "name": "balance", "constant": true }, + {"type": "function", "name": "send", "constant": false, "inputs": [{ "name": "amount", "type": "uint256" }]}, + {"type": "function", "name": "transfer", "constant": false, "inputs": [{"name": "from", "type": "address"}, {"name": "to", "type": "address"}, {"name": "value", "type": "uint256"}], "outputs": [{"name": "success", "type": "bool"}]}, + {"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple"}],"name":"tuple","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}, + {"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[]"}],"name":"tupleSlice","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}, + {"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[5]"}],"name":"tupleArray","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"}, + {"constant":false,"inputs":[{"components":[{"name":"x","type":"uint256"},{"name":"y","type":"uint256"}],"name":"a","type":"tuple[5][]"}],"name":"complexTuple","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"} ]` func TestMethodString(t *testing.T) { @@ -45,6 +49,22 @@ func TestMethodString(t *testing.T) { method: "transfer", expectation: "function transfer(address from, address to, uint256 value) returns(bool success)", }, + { + method: "tuple", + expectation: "function tuple((uint256,uint256) a) returns()", + }, + { + method: "tupleArray", + expectation: "function tupleArray((uint256,uint256)[5] a) returns()", + }, + { + method: "tupleSlice", + expectation: "function tupleSlice((uint256,uint256)[] a) returns()", + }, + { + method: "complexTuple", + expectation: "function complexTuple((uint256,uint256)[5][] a) returns()", + }, } abi, err := JSON(strings.NewReader(methoddata)) @@ -59,3 +79,50 @@ func TestMethodString(t *testing.T) { } } } + +func TestMethodSig(t *testing.T) { + var cases = []struct { + method string + expect string + }{ + { + method: "balance", + expect: "balance()", + }, + { + method: "send", + expect: "send(uint256)", + }, + { + method: "transfer", + expect: "transfer(address,address,uint256)", + }, + { + method: "tuple", + expect: "tuple((uint256,uint256))", + }, + { + method: "tupleArray", + expect: "tupleArray((uint256,uint256)[5])", + }, + { + method: "tupleSlice", + expect: "tupleSlice((uint256,uint256)[])", + }, + { + method: "complexTuple", + expect: "complexTuple((uint256,uint256)[5][])", + }, + } + abi, err := JSON(strings.NewReader(methoddata)) + if err != nil { + t.Fatal(err) + } + + for _, test := range cases { + got := abi.Methods[test.method].Sig() + if got != test.expect { + t.Errorf("expected string to be %s, got %s", test.expect, got) + } + } +} diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index ccc6a65932..73ca8fa2bd 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -31,6 +31,14 @@ func indirect(v reflect.Value) reflect.Value { return v } +// indirectInterfaceOrPtr recursively dereferences the value until value is not interface. +func indirectInterfaceOrPtr(v reflect.Value) reflect.Value { + if (v.Kind() == reflect.Interface || v.Kind() == reflect.Ptr) && v.Elem().IsValid() { + return indirect(v.Elem()) + } + return v +} + // reflectIntKind returns the reflect using the given size and // unsignedness. func reflectIntKindAndType(unsigned bool, size int) (reflect.Kind, reflect.Type) { diff --git a/accounts/abi/type.go b/accounts/abi/type.go index 1a37182353..597d314392 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -68,7 +68,6 @@ func NewType(t string, components []ArgumentMarshaling) (typ Type, err error) { if strings.Count(t, "[") != strings.Count(t, "]") { return Type{}, fmt.Errorf("invalid arg type in abi") } - typ.stringKind = t // if there are brackets, get ready to go into slice/array mode and @@ -92,9 +91,7 @@ func NewType(t string, components []ArgumentMarshaling) (typ Type, err error) { typ.Kind = reflect.Slice typ.Elem = &embeddedType typ.Type = reflect.SliceOf(embeddedType.Type) - if embeddedType.T == TupleTy { - typ.stringKind = embeddedType.stringKind + sliced - } + typ.stringKind = embeddedType.stringKind + sliced } else if len(intz) == 1 { // is a array typ.T = ArrayTy @@ -105,9 +102,7 @@ func NewType(t string, components []ArgumentMarshaling) (typ Type, err error) { return Type{}, fmt.Errorf("abi: error parsing variable size: %v", err) } typ.Type = reflect.ArrayOf(typ.Size, embeddedType.Type) - if embeddedType.T == TupleTy { - typ.stringKind = embeddedType.stringKind + sliced - } + typ.stringKind = embeddedType.stringKind + sliced } else { return Type{}, fmt.Errorf("invalid formatting of array type") } diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index fa8a69d05c..c85b86d8c0 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -965,25 +965,21 @@ func TestUnpackTuple(t *testing.T) { buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000001")) // ret[a] = 1 buff.Write(common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) // ret[b] = -1 + // If the result is single tuple, use struct as return value container directly. v := struct { - Ret struct { - A *big.Int - B *big.Int - } - }{Ret: struct { A *big.Int B *big.Int - }{new(big.Int), new(big.Int)}} + }{new(big.Int), new(big.Int)} err = abi.Unpack(&v, "tuple", buff.Bytes()) if err != nil { t.Error(err) } else { - if v.Ret.A.Cmp(big.NewInt(1)) != 0 { - t.Errorf("unexpected value unpacked: want %x, got %x", 1, v.Ret.A) + if v.A.Cmp(big.NewInt(1)) != 0 { + t.Errorf("unexpected value unpacked: want %x, got %x", 1, v.A) } - if v.Ret.B.Cmp(big.NewInt(-1)) != 0 { - t.Errorf("unexpected value unpacked: want %x, got %x", v.Ret.B, -1) + if v.B.Cmp(big.NewInt(-1)) != 0 { + t.Errorf("unexpected value unpacked: want %x, got %x", v.B, -1) } } diff --git a/accounts/external/backend.go b/accounts/external/backend.go index 23037f52d1..705c987227 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -182,18 +182,21 @@ func (api *ExternalSigner) SignText(account accounts.Account, text []byte) ([]by func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { res := ethapi.SignTransactionResult{} - to := common.NewMixedcaseAddress(*tx.To()) data := hexutil.Bytes(tx.Data()) + var to *common.MixedcaseAddress + if tx.To() != nil { + t := common.NewMixedcaseAddress(*tx.To()) + to = &t + } args := &core.SendTxArgs{ Data: &data, Nonce: hexutil.Uint64(tx.Nonce()), Value: hexutil.Big(*tx.Value()), Gas: hexutil.Uint64(tx.Gas()), GasPrice: hexutil.Big(*tx.GasPrice()), - To: &to, + To: to, From: common.NewMixedcaseAddress(account.Address), } - if err := api.client.Call(&res, "account_signTransaction", args); err != nil { return nil, err } diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index 7ef2e69895..f9e0486238 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -69,6 +69,8 @@ func main() { lang = bind.LangGo case "java": lang = bind.LangJava + case "objc": + lang = bind.LangObjC default: fmt.Printf("Unsupported destination language \"%s\" (--lang)\n", *langFlag) os.Exit(-1) diff --git a/cmd/checkpoint-admin/common.go b/cmd/checkpoint-admin/common.go index 1f4a34a414..107cd1de02 100644 --- a/cmd/checkpoint-admin/common.go +++ b/cmd/checkpoint-admin/common.go @@ -17,14 +17,13 @@ package main import ( - "io/ioutil" "strconv" - "strings" - "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/external" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/console" "github.com/ethereum/go-ethereum/contracts/checkpointoracle" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/params" @@ -111,56 +110,11 @@ func newContract(client *rpc.Client) (common.Address, *checkpointoracle.Checkpoi return addr, contract } -// promptPassphrase prompts the user for a passphrase. -// Set confirmation to true to require the user to confirm the passphrase. -func promptPassphrase(confirmation bool) string { - passphrase, err := console.Stdin.PromptPassword("Passphrase: ") +// newClefSigner sets up a clef backend and returns a clef transaction signer. +func newClefSigner(ctx *cli.Context) *bind.TransactOpts { + clef, err := external.NewExternalSigner(ctx.String(clefURLFlag.Name)) if err != nil { - utils.Fatalf("Failed to read passphrase: %v", err) + utils.Fatalf("Failed to create clef signer %v", err) } - - if confirmation { - confirm, err := console.Stdin.PromptPassword("Repeat passphrase: ") - if err != nil { - utils.Fatalf("Failed to read passphrase confirmation: %v", err) - } - if passphrase != confirm { - utils.Fatalf("Passphrases do not match") - } - } - return passphrase -} - -// getPassphrase obtains a passphrase given by the user. It first checks the -// --password command line flag and ultimately prompts the user for a -// passphrase. -func getPassphrase(ctx *cli.Context) string { - passphraseFile := ctx.String(utils.PasswordFileFlag.Name) - if passphraseFile != "" { - content, err := ioutil.ReadFile(passphraseFile) - if err != nil { - utils.Fatalf("Failed to read passphrase file '%s': %v", - passphraseFile, err) - } - return strings.TrimRight(string(content), "\r\n") - } - // Otherwise prompt the user for the passphrase. - return promptPassphrase(false) -} - -// getKey retrieves the user key through specified key file. -func getKey(ctx *cli.Context) *keystore.Key { - // Read key from file. - keyFile := ctx.GlobalString(keyFileFlag.Name) - keyJson, err := ioutil.ReadFile(keyFile) - if err != nil { - utils.Fatalf("Failed to read the keyfile at '%s': %v", keyFile, err) - } - // Decrypt key with passphrase. - passphrase := getPassphrase(ctx) - key, err := keystore.DecryptKey(keyJson, passphrase) - if err != nil { - utils.Fatalf("Failed to decrypt user key '%s': %v", keyFile, err) - } - return key + return bind.NewClefTransactor(clef, accounts.Account{Address: common.HexToAddress(ctx.String(signerFlag.Name))}) } diff --git a/cmd/checkpoint-admin/exec.go b/cmd/checkpoint-admin/exec.go index 02c4f35cc5..1ce975f494 100644 --- a/cmd/checkpoint-admin/exec.go +++ b/cmd/checkpoint-admin/exec.go @@ -26,7 +26,6 @@ import ( "time" "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -46,10 +45,9 @@ var commandDeploy = cli.Command{ Flags: []cli.Flag{ nodeURLFlag, clefURLFlag, + signerFlag, signersFlag, thresholdFlag, - keyFileFlag, - utils.PasswordFileFlag, }, Action: utils.MigrateFlags(deploy), } @@ -60,12 +58,10 @@ var commandSign = cli.Command{ Flags: []cli.Flag{ nodeURLFlag, clefURLFlag, + signerFlag, indexFlag, hashFlag, oracleFlag, - keyFileFlag, - signerFlag, - utils.PasswordFileFlag, }, Action: utils.MigrateFlags(sign), } @@ -75,10 +71,10 @@ var commandPublish = cli.Command{ Usage: "Publish a checkpoint into the oracle", Flags: []cli.Flag{ nodeURLFlag, + clefURLFlag, + signerFlag, indexFlag, signaturesFlag, - keyFileFlag, - utils.PasswordFileFlag, }, Action: utils.MigrateFlags(publish), } @@ -108,11 +104,11 @@ func deploy(ctx *cli.Context) error { } fmt.Printf("\nSignatures needed to publish: %d\n", needed) - // Retrieve the private key, create an abigen transactor and an RPC client - transactor := bind.NewKeyedTransactor(getKey(ctx).PrivateKey) - client := newClient(ctx) + // setup clef signer, create an abigen transactor and an RPC client + transactor, client := newClefSigner(ctx), newClient(ctx) // Deploy the checkpoint oracle + fmt.Println("Sending deploy request to Clef...") oracle, tx, _, err := contract.DeployCheckpointOracle(transactor, client, addrs, big.NewInt(int64(params.CheckpointFrequency)), big.NewInt(int64(params.CheckpointProcessConfirmations)), big.NewInt(int64(needed))) if err != nil { @@ -158,9 +154,7 @@ func sign(ctx *cli.Context) error { node = newRPCClient(ctx.GlobalString(nodeURLFlag.Name)) checkpoint := getCheckpoint(ctx, node) - chash = checkpoint.Hash() - cindex = checkpoint.SectionIndex - address = getContractAddr(node) + chash, cindex, address = checkpoint.Hash(), checkpoint.SectionIndex, getContractAddr(node) // Check the validity of checkpoint reqCtx, cancelFn := context.WithTimeout(context.Background(), 10*time.Second) @@ -207,43 +201,24 @@ func sign(ctx *cli.Context) error { fmt.Printf("Oracle => %s\n", address.Hex()) fmt.Printf("Index %4d => %s\n", cindex, chash.Hex()) - switch { - case ctx.GlobalIsSet(clefURLFlag.Name): - // Sign checkpoint in clef mode. - signer = ctx.String(signerFlag.Name) + // Sign checkpoint in clef mode. + signer = ctx.String(signerFlag.Name) - if !offline { - if err := isAdmin(common.HexToAddress(signer)); err != nil { - return err - } + if !offline { + if err := isAdmin(common.HexToAddress(signer)); err != nil { + return err } - clef := newRPCClient(ctx.GlobalString(clefURLFlag.Name)) - p := make(map[string]string) - buf := make([]byte, 8) - binary.BigEndian.PutUint64(buf, cindex) - p["address"] = address.Hex() - p["message"] = hexutil.Encode(append(buf, chash.Bytes()...)) - if err := clef.Call(&signature, "account_signData", accounts.MimetypeDataWithValidator, signer, p); err != nil { - utils.Fatalf("Failed to sign checkpoint, err %v", err) - } - case ctx.GlobalIsSet(keyFileFlag.Name): - // Sign checkpoint in raw private key file mode. - key := getKey(ctx) - signer = key.Address.Hex() + } + clef := newRPCClient(ctx.String(clefURLFlag.Name)) + p := make(map[string]string) + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, cindex) + p["address"] = address.Hex() + p["message"] = hexutil.Encode(append(buf, chash.Bytes()...)) - if !offline { - if err := isAdmin(key.Address); err != nil { - return err - } - } - sig, err := crypto.Sign(sighash(cindex, address, chash), key.PrivateKey) - if err != nil { - utils.Fatalf("Failed to sign checkpoint, err %v", err) - } - sig[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper - signature = common.Bytes2Hex(sig) - default: - utils.Fatalf("Please specify clef URL or private key file path to sign checkpoint") + fmt.Println("Sending signing request to Clef...") + if err := clef.Call(&signature, "account_signData", accounts.MimetypeDataWithValidator, signer, p); err != nil { + utils.Fatalf("Failed to sign checkpoint, err %v", err) } fmt.Printf("Signer => %s\n", signer) fmt.Printf("Signature => %s\n", signature) @@ -326,7 +301,8 @@ func publish(ctx *cli.Context) error { fmt.Printf("Sentry number => %d\nSentry hash => %s\n", recent.Number, recent.Hash().Hex()) // Publish the checkpoint into the oracle - tx, err := oracle.RegisterCheckpoint(getKey(ctx).PrivateKey, checkpoint.SectionIndex, checkpoint.Hash().Bytes(), recent.Number, recent.Hash(), sigs) + fmt.Println("Sending publish request to Clef...") + tx, err := oracle.RegisterCheckpoint(newClefSigner(ctx), checkpoint.SectionIndex, checkpoint.Hash().Bytes(), recent.Number, recent.Hash(), sigs) if err != nil { utils.Fatalf("Register contract failed %v", err) } diff --git a/cmd/checkpoint-admin/main.go b/cmd/checkpoint-admin/main.go index 39eae6878e..1fdec60a0c 100644 --- a/cmd/checkpoint-admin/main.go +++ b/cmd/checkpoint-admin/main.go @@ -59,10 +59,7 @@ func init() { } app.Flags = []cli.Flag{ oracleFlag, - keyFileFlag, nodeURLFlag, - clefURLFlag, - utils.PasswordFileFlag, } cli.CommandHelpTemplate = commandHelperTemplate } @@ -85,10 +82,6 @@ var ( Name: "threshold", Usage: "Minimal number of signatures required to approve a checkpoint", } - keyFileFlag = cli.StringFlag{ - Name: "keyfile", - Usage: "The private key file (keyfile signature is not recommended)", - } nodeURLFlag = cli.StringFlag{ Name: "rpc", Value: "http://localhost:8545", @@ -101,7 +94,7 @@ var ( } signerFlag = cli.StringFlag{ Name: "signer", - Usage: "Signer address for clef mode signing", + Usage: "Signer address for clef signing", } signersFlag = cli.StringFlag{ Name: "signers", diff --git a/contracts/checkpointoracle/oracle.go b/contracts/checkpointoracle/oracle.go index 702e27d959..13ff236f2d 100644 --- a/contracts/checkpointoracle/oracle.go +++ b/contracts/checkpointoracle/oracle.go @@ -20,7 +20,6 @@ package checkpointoracle //go:generate abigen --sol contract/oracle.sol --pkg contract --out contract/oracle.go import ( - "crypto/ecdsa" "errors" "math/big" @@ -73,7 +72,7 @@ func (oracle *CheckpointOracle) LookupCheckpointEvents(blockLogs [][]*types.Log, // // Notably all signatures given should be transformed to "ethereum style" which transforms // v from 0/1 to 27/28 according to the yellow paper. -func (oracle *CheckpointOracle) RegisterCheckpoint(key *ecdsa.PrivateKey, index uint64, hash []byte, rnum *big.Int, rhash [32]byte, sigs [][]byte) (*types.Transaction, error) { +func (oracle *CheckpointOracle) RegisterCheckpoint(opts *bind.TransactOpts, index uint64, hash []byte, rnum *big.Int, rhash [32]byte, sigs [][]byte) (*types.Transaction, error) { var ( r [][32]byte s [][32]byte @@ -87,5 +86,5 @@ func (oracle *CheckpointOracle) RegisterCheckpoint(key *ecdsa.PrivateKey, index s = append(s, common.BytesToHash(sigs[i][32:64])) v = append(v, sigs[i][64]) } - return oracle.contract.SetCheckpoint(bind.NewKeyedTransactor(key), rnum, rhash, common.BytesToHash(hash), index, v, r, s) + return oracle.contract.SetCheckpoint(opts, rnum, rhash, common.BytesToHash(hash), index, v, r, s) } diff --git a/core/blockchain.go b/core/blockchain.go index 95d09dad13..1330fab933 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -930,6 +930,8 @@ func (bc *BlockChain) truncateAncient(head uint64) error { // InsertReceiptChain attempts to complete an already existing header chain with // transaction and receipt data. func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts, ancientLimit uint64) (int, error) { + // We don't require the chainMu here since we want to maximize the + // concurrency of header insertion and receipt insertion. bc.wg.Add(1) defer bc.wg.Done() @@ -962,19 +964,21 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // updateHead updates the head fast sync block if the inserted blocks are better // and returns a indicator whether the inserted blocks are canonical. updateHead := func(head *types.Block) bool { - var isCanonical bool bc.chainmu.Lock() - if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case - currentFastBlock := bc.CurrentFastBlock() + + // Rewind may have occurred, skip in that case. + if bc.CurrentHeader().Number.Cmp(head.Number()) >= 0 { + currentFastBlock, td := bc.CurrentFastBlock(), bc.GetTd(head.Hash(), head.NumberU64()) if bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()).Cmp(td) < 0 { rawdb.WriteHeadFastBlockHash(bc.db, head.Hash()) bc.currentFastBlock.Store(head) headFastBlockGauge.Update(int64(head.NumberU64())) - isCanonical = true + bc.chainmu.Unlock() + return true } } bc.chainmu.Unlock() - return isCanonical + return false } // writeAncient writes blockchain and corresponding receipt chain into ancient store. // diff --git a/core/chain_indexer.go b/core/chain_indexer.go index c82febb545..c0c2c4f7f9 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -418,7 +418,7 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com // actual canonical chain and rolls back reorged sections if necessary to ensure that stored // sections are all valid func (c *ChainIndexer) verifyLastHead() { - for c.storedSections > 0 { + for c.storedSections > 0 && c.storedSections > c.checkpointSections { if c.SectionHead(c.storedSections-1) == rawdb.ReadCanonicalHash(c.chainDb, c.storedSections*c.sectionSize-1) { return } diff --git a/les/api.go b/les/api.go index b53512196f..95e1b009e5 100644 --- a/les/api.go +++ b/les/api.go @@ -17,462 +17,16 @@ package les import ( - "context" "errors" - "fmt" - "sync" - "time" "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/common/mclock" - "github.com/ethereum/go-ethereum/les/csvlogger" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/rpc" ) var ( - ErrMinCap = errors.New("capacity too small") - ErrTotalCap = errors.New("total capacity exceeded") - ErrUnknownBenchmarkType = errors.New("unknown benchmark type") - ErrNoCheckpoint = errors.New("no local checkpoint provided") - ErrNotActivated = errors.New("checkpoint registrar is not activated") - - dropCapacityDelay = time.Second // delay applied to decreasing capacity changes + errNoCheckpoint = errors.New("no local checkpoint provided") + errNotActivated = errors.New("checkpoint registrar is not activated") ) -// PrivateLightServerAPI provides an API to access the LES light server. -// It offers only methods that operate on public data that is freely available to anyone. -type PrivateLightServerAPI struct { - server *LesServer -} - -// NewPrivateLightServerAPI creates a new LES light server API. -func NewPrivateLightServerAPI(server *LesServer) *PrivateLightServerAPI { - return &PrivateLightServerAPI{ - server: server, - } -} - -// TotalCapacity queries total available capacity for all clients -func (api *PrivateLightServerAPI) TotalCapacity() hexutil.Uint64 { - return hexutil.Uint64(api.server.priorityClientPool.totalCapacity()) -} - -// SubscribeTotalCapacity subscribes to changed total capacity events. -// If onlyUnderrun is true then notification is sent only if the total capacity -// drops under the total capacity of connected priority clients. -// -// Note: actually applying decreasing total capacity values is delayed while the -// notification is sent instantly. This allows lowering the capacity of a priority client -// or choosing which one to drop before the system drops some of them automatically. -func (api *PrivateLightServerAPI) SubscribeTotalCapacity(ctx context.Context, onlyUnderrun bool) (*rpc.Subscription, error) { - notifier, supported := rpc.NotifierFromContext(ctx) - if !supported { - return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported - } - rpcSub := notifier.CreateSubscription() - api.server.priorityClientPool.subscribeTotalCapacity(&tcSubscription{notifier, rpcSub, onlyUnderrun}) - return rpcSub, nil -} - -type ( - // tcSubscription represents a total capacity subscription - tcSubscription struct { - notifier *rpc.Notifier - rpcSub *rpc.Subscription - onlyUnderrun bool - } - tcSubs map[*tcSubscription]struct{} -) - -// send sends a changed total capacity event to the subscribers -func (s tcSubs) send(tc uint64, underrun bool) { - for sub := range s { - select { - case <-sub.rpcSub.Err(): - delete(s, sub) - case <-sub.notifier.Closed(): - delete(s, sub) - default: - if underrun || !sub.onlyUnderrun { - sub.notifier.Notify(sub.rpcSub.ID, tc) - } - } - } -} - -// MinimumCapacity queries minimum assignable capacity for a single client -func (api *PrivateLightServerAPI) MinimumCapacity() hexutil.Uint64 { - return hexutil.Uint64(api.server.minCapacity) -} - -// FreeClientCapacity queries the capacity provided for free clients -func (api *PrivateLightServerAPI) FreeClientCapacity() hexutil.Uint64 { - return hexutil.Uint64(api.server.freeClientCap) -} - -// SetClientCapacity sets the priority capacity assigned to a given client. -// If the assigned capacity is bigger than zero then connection is always -// guaranteed. The sum of capacity assigned to priority clients can not exceed -// the total available capacity. -// -// Note: assigned capacity can be changed while the client is connected with -// immediate effect. -func (api *PrivateLightServerAPI) SetClientCapacity(id enode.ID, cap uint64) error { - if cap != 0 && cap < api.server.minCapacity { - return ErrMinCap - } - return api.server.priorityClientPool.setClientCapacity(id, cap) -} - -// GetClientCapacity returns the capacity assigned to a given client -func (api *PrivateLightServerAPI) GetClientCapacity(id enode.ID) hexutil.Uint64 { - api.server.priorityClientPool.lock.Lock() - defer api.server.priorityClientPool.lock.Unlock() - - return hexutil.Uint64(api.server.priorityClientPool.clients[id].cap) -} - -// clientPool is implemented by both the free and priority client pools -type clientPool interface { - peerSetNotify - setLimits(count int, totalCap uint64) -} - -// priorityClientPool stores information about prioritized clients -type priorityClientPool struct { - lock sync.Mutex - child clientPool - ps *peerSet - clients map[enode.ID]priorityClientInfo - totalCap, totalCapAnnounced uint64 - totalConnectedCap, freeClientCap uint64 - maxPeers, priorityCount int - logger *csvlogger.Logger - logTotalPriConn *csvlogger.Channel - - subs tcSubs - updateSchedule []scheduledUpdate - scheduleCounter uint64 -} - -// scheduledUpdate represents a delayed total capacity update -type scheduledUpdate struct { - time mclock.AbsTime - totalCap, id uint64 -} - -// priorityClientInfo entries exist for all prioritized clients and currently connected non-priority clients -type priorityClientInfo struct { - cap uint64 // zero for non-priority clients - connected bool - peer *peer -} - -// newPriorityClientPool creates a new priority client pool -func newPriorityClientPool(freeClientCap uint64, ps *peerSet, child clientPool, metricsLogger, eventLogger *csvlogger.Logger) *priorityClientPool { - return &priorityClientPool{ - clients: make(map[enode.ID]priorityClientInfo), - freeClientCap: freeClientCap, - ps: ps, - child: child, - logger: eventLogger, - logTotalPriConn: metricsLogger.NewChannel("totalPriConn", 0), - } -} - -// registerPeer is called when a new client is connected. If the client has no -// priority assigned then it is passed to the child pool which may either keep it -// or disconnect it. -// -// Note: priorityClientPool also stores a record about free clients while they are -// connected in order to be able to assign priority to them later. -func (v *priorityClientPool) registerPeer(p *peer) { - v.lock.Lock() - defer v.lock.Unlock() - - id := p.ID() - c := v.clients[id] - v.logger.Event(fmt.Sprintf("priorityClientPool: registerPeer cap=%d connected=%v, %x", c.cap, c.connected, id.Bytes())) - if c.connected { - return - } - if c.cap == 0 && v.child != nil { - v.child.registerPeer(p) - } - if c.cap != 0 && v.totalConnectedCap+c.cap > v.totalCap { - v.logger.Event(fmt.Sprintf("priorityClientPool: rejected, %x", id.Bytes())) - go v.ps.Unregister(p.id) - return - } - - c.connected = true - c.peer = p - v.clients[id] = c - if c.cap != 0 { - v.priorityCount++ - v.totalConnectedCap += c.cap - v.logger.Event(fmt.Sprintf("priorityClientPool: accepted with %d capacity, %x", c.cap, id.Bytes())) - v.logTotalPriConn.Update(float64(v.totalConnectedCap)) - if v.child != nil { - v.child.setLimits(v.maxPeers-v.priorityCount, v.totalCap-v.totalConnectedCap) - } - p.updateCapacity(c.cap) - } -} - -// unregisterPeer is called when a client is disconnected. If the client has no -// priority assigned then it is also removed from the child pool. -func (v *priorityClientPool) unregisterPeer(p *peer) { - v.lock.Lock() - defer v.lock.Unlock() - - id := p.ID() - c := v.clients[id] - v.logger.Event(fmt.Sprintf("priorityClientPool: unregisterPeer cap=%d connected=%v, %x", c.cap, c.connected, id.Bytes())) - if !c.connected { - return - } - if c.cap != 0 { - c.connected = false - v.clients[id] = c - v.priorityCount-- - v.totalConnectedCap -= c.cap - v.logTotalPriConn.Update(float64(v.totalConnectedCap)) - if v.child != nil { - v.child.setLimits(v.maxPeers-v.priorityCount, v.totalCap-v.totalConnectedCap) - } - } else { - if v.child != nil { - v.child.unregisterPeer(p) - } - delete(v.clients, id) - } -} - -// setLimits updates the allowed peer count and total capacity of the priority -// client pool. Since the free client pool is a child of the priority pool the -// remaining peer count and capacity is assigned to the free pool by calling its -// own setLimits function. -// -// Note: a decreasing change of the total capacity is applied with a delay. -func (v *priorityClientPool) setLimits(count int, totalCap uint64) { - v.lock.Lock() - defer v.lock.Unlock() - - v.totalCapAnnounced = totalCap - if totalCap > v.totalCap { - v.setLimitsNow(count, totalCap) - v.subs.send(totalCap, false) - return - } - v.setLimitsNow(count, v.totalCap) - if totalCap < v.totalCap { - v.subs.send(totalCap, totalCap < v.totalConnectedCap) - for i, s := range v.updateSchedule { - if totalCap >= s.totalCap { - s.totalCap = totalCap - v.updateSchedule = v.updateSchedule[:i+1] - return - } - } - v.updateSchedule = append(v.updateSchedule, scheduledUpdate{time: mclock.Now() + mclock.AbsTime(dropCapacityDelay), totalCap: totalCap}) - if len(v.updateSchedule) == 1 { - v.scheduleCounter++ - id := v.scheduleCounter - v.updateSchedule[0].id = id - time.AfterFunc(dropCapacityDelay, func() { v.checkUpdate(id) }) - } - } else { - v.updateSchedule = nil - } -} - -// checkUpdate performs the next scheduled update if possible and schedules -// the one after that -func (v *priorityClientPool) checkUpdate(id uint64) { - v.lock.Lock() - defer v.lock.Unlock() - - if len(v.updateSchedule) == 0 || v.updateSchedule[0].id != id { - return - } - v.setLimitsNow(v.maxPeers, v.updateSchedule[0].totalCap) - v.updateSchedule = v.updateSchedule[1:] - if len(v.updateSchedule) != 0 { - v.scheduleCounter++ - id := v.scheduleCounter - v.updateSchedule[0].id = id - dt := time.Duration(v.updateSchedule[0].time - mclock.Now()) - time.AfterFunc(dt, func() { v.checkUpdate(id) }) - } -} - -// setLimits updates the allowed peer count and total capacity immediately -func (v *priorityClientPool) setLimitsNow(count int, totalCap uint64) { - if v.priorityCount > count || v.totalConnectedCap > totalCap { - for id, c := range v.clients { - if c.connected { - v.logger.Event(fmt.Sprintf("priorityClientPool: setLimitsNow kicked out, %x", id.Bytes())) - c.connected = false - v.totalConnectedCap -= c.cap - v.logTotalPriConn.Update(float64(v.totalConnectedCap)) - v.priorityCount-- - v.clients[id] = c - go v.ps.Unregister(c.peer.id) - if v.priorityCount <= count && v.totalConnectedCap <= totalCap { - break - } - } - } - } - v.maxPeers = count - v.totalCap = totalCap - if v.child != nil { - v.child.setLimits(v.maxPeers-v.priorityCount, v.totalCap-v.totalConnectedCap) - } -} - -// totalCapacity queries total available capacity for all clients -func (v *priorityClientPool) totalCapacity() uint64 { - v.lock.Lock() - defer v.lock.Unlock() - - return v.totalCapAnnounced -} - -// subscribeTotalCapacity subscribes to changed total capacity events -func (v *priorityClientPool) subscribeTotalCapacity(sub *tcSubscription) { - v.lock.Lock() - defer v.lock.Unlock() - - v.subs[sub] = struct{}{} -} - -// setClientCapacity sets the priority capacity assigned to a given client -func (v *priorityClientPool) setClientCapacity(id enode.ID, cap uint64) error { - v.lock.Lock() - defer v.lock.Unlock() - - c := v.clients[id] - if c.cap == cap { - return nil - } - if c.connected { - if v.totalConnectedCap+cap > v.totalCap+c.cap { - return ErrTotalCap - } - if c.cap == 0 { - if v.child != nil { - v.child.unregisterPeer(c.peer) - } - v.priorityCount++ - } - if cap == 0 { - v.priorityCount-- - } - v.totalConnectedCap += cap - c.cap - v.logTotalPriConn.Update(float64(v.totalConnectedCap)) - if v.child != nil { - v.child.setLimits(v.maxPeers-v.priorityCount, v.totalCap-v.totalConnectedCap) - } - if cap == 0 { - if v.child != nil { - v.child.registerPeer(c.peer) - } - c.peer.updateCapacity(v.freeClientCap) - } else { - c.peer.updateCapacity(cap) - } - } - if cap != 0 || c.connected { - c.cap = cap - v.clients[id] = c - } else { - delete(v.clients, id) - } - if c.connected { - v.logger.Event(fmt.Sprintf("priorityClientPool: changed capacity to %d, %x", cap, id.Bytes())) - } - return nil -} - -// Benchmark runs a request performance benchmark with a given set of measurement setups -// in multiple passes specified by passCount. The measurement time for each setup in each -// pass is specified in milliseconds by length. -// -// Note: measurement time is adjusted for each pass depending on the previous ones. -// Therefore a controlled total measurement time is achievable in multiple passes. -func (api *PrivateLightServerAPI) Benchmark(setups []map[string]interface{}, passCount, length int) ([]map[string]interface{}, error) { - benchmarks := make([]requestBenchmark, len(setups)) - for i, setup := range setups { - if t, ok := setup["type"].(string); ok { - getInt := func(field string, def int) int { - if value, ok := setup[field].(float64); ok { - return int(value) - } - return def - } - getBool := func(field string, def bool) bool { - if value, ok := setup[field].(bool); ok { - return value - } - return def - } - switch t { - case "header": - benchmarks[i] = &benchmarkBlockHeaders{ - amount: getInt("amount", 1), - skip: getInt("skip", 1), - byHash: getBool("byHash", false), - reverse: getBool("reverse", false), - } - case "body": - benchmarks[i] = &benchmarkBodiesOrReceipts{receipts: false} - case "receipts": - benchmarks[i] = &benchmarkBodiesOrReceipts{receipts: true} - case "proof": - benchmarks[i] = &benchmarkProofsOrCode{code: false} - case "code": - benchmarks[i] = &benchmarkProofsOrCode{code: true} - case "cht": - benchmarks[i] = &benchmarkHelperTrie{ - bloom: false, - reqCount: getInt("amount", 1), - } - case "bloom": - benchmarks[i] = &benchmarkHelperTrie{ - bloom: true, - reqCount: getInt("amount", 1), - } - case "txSend": - benchmarks[i] = &benchmarkTxSend{} - case "txStatus": - benchmarks[i] = &benchmarkTxStatus{} - default: - return nil, ErrUnknownBenchmarkType - } - } else { - return nil, ErrUnknownBenchmarkType - } - } - rs := api.server.protocolManager.runBenchmark(benchmarks, passCount, time.Millisecond*time.Duration(length)) - result := make([]map[string]interface{}, len(setups)) - for i, r := range rs { - res := make(map[string]interface{}) - if r.err == nil { - res["totalCount"] = r.totalCount - res["avgTime"] = r.avgTime - res["maxInSize"] = r.maxInSize - res["maxOutSize"] = r.maxOutSize - } else { - res["error"] = r.err.Error() - } - result[i] = res - } - return result, nil -} - // PrivateLightAPI provides an API to access the LES light server or light client. type PrivateLightAPI struct { backend *lesCommons @@ -498,7 +52,7 @@ func (api *PrivateLightAPI) LatestCheckpoint() ([4]string, error) { var res [4]string cp := api.backend.latestLocalCheckpoint() if cp.Empty() { - return res, ErrNoCheckpoint + return res, errNoCheckpoint } res[0] = hexutil.EncodeUint64(cp.SectionIndex) res[1], res[2], res[3] = cp.SectionHead.Hex(), cp.CHTRoot.Hex(), cp.BloomRoot.Hex() @@ -515,7 +69,7 @@ func (api *PrivateLightAPI) GetCheckpoint(index uint64) ([3]string, error) { var res [3]string cp := api.backend.getLocalCheckpoint(index) if cp.Empty() { - return res, ErrNoCheckpoint + return res, errNoCheckpoint } res[0], res[1], res[2] = cp.SectionHead.Hex(), cp.CHTRoot.Hex(), cp.BloomRoot.Hex() return res, nil @@ -524,7 +78,7 @@ func (api *PrivateLightAPI) GetCheckpoint(index uint64) ([3]string, error) { // GetCheckpointContractAddress returns the contract contract address in hex format. func (api *PrivateLightAPI) GetCheckpointContractAddress() (string, error) { if api.reg == nil { - return "", ErrNotActivated + return "", errNotActivated } return api.reg.config.Address.Hex(), nil } diff --git a/les/costtracker.go b/les/costtracker.go index e463c9f8b0..2d9c95af78 100644 --- a/les/costtracker.go +++ b/les/costtracker.go @@ -18,7 +18,6 @@ package les import ( "encoding/binary" - "fmt" "math" "sync" "sync/atomic" @@ -27,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/les/csvlogger" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/log" ) @@ -96,40 +94,50 @@ const ( // as the number of cost units per nanosecond of serving time in a single thread. // It is based on statistics collected during serving requests in high-load periods // and practically acts as a one-dimension request price scaling factor over the -// pre-defined cost estimate table. Instead of scaling the cost values, the real -// value of cost units is changed by applying the factor to the serving times. This -// is more convenient because the changes in the cost factor can be applied immediately -// without always notifying the clients about the changed cost tables. +// pre-defined cost estimate table. +// +// The reason for dynamically maintaining the global factor on the server side is: +// the estimated time cost of the request is fixed(hardcoded) but the configuration +// of the machine running the server is really different. Therefore, the request serving +// time in different machine will vary greatly. And also, the request serving time +// in same machine may vary greatly with different request pressure. +// +// In order to more effectively limit resources, we apply the global factor to serving +// time to make the result as close as possible to the estimated time cost no matter +// the server is slow or fast. And also we scale the totalRecharge with global factor +// so that fast server can serve more requests than estimation and slow server can +// reduce request pressure. +// +// Instead of scaling the cost values, the real value of cost units is changed by +// applying the factor to the serving times. This is more convenient because the +// changes in the cost factor can be applied immediately without always notifying +// the clients about the changed cost tables. type costTracker struct { db ethdb.Database stopCh chan chan struct{} - inSizeFactor, outSizeFactor float64 - gf, utilTarget float64 - minBufLimit uint64 + inSizeFactor float64 + outSizeFactor float64 + factor float64 + utilTarget float64 + minBufLimit uint64 - gfUpdateCh chan gfUpdate gfLock sync.RWMutex + reqInfoCh chan reqInfo totalRechargeCh chan uint64 - stats map[uint64][]uint64 - logger *csvlogger.Logger - logRecentTime, logRecentAvg, logTotalRecharge, logRelCost *csvlogger.Channel + stats map[uint64][]uint64 // Used for testing purpose. } // newCostTracker creates a cost tracker and loads the cost factor statistics from the database. // It also returns the minimum capacity that can be assigned to any peer. -func newCostTracker(db ethdb.Database, config *eth.Config, logger *csvlogger.Logger) (*costTracker, uint64) { +func newCostTracker(db ethdb.Database, config *eth.Config) (*costTracker, uint64) { utilTarget := float64(config.LightServ) * flowcontrol.FixedPointMultiplier / 100 ct := &costTracker{ - db: db, - stopCh: make(chan chan struct{}), - utilTarget: utilTarget, - logger: logger, - logRelCost: logger.NewMinMaxChannel("relativeCost", true), - logRecentTime: logger.NewMinMaxChannel("recentTime", true), - logRecentAvg: logger.NewMinMaxChannel("recentAvg", true), - logTotalRecharge: logger.NewChannel("totalRecharge", 0.01), + db: db, + stopCh: make(chan chan struct{}), + reqInfoCh: make(chan reqInfo, 100), + utilTarget: utilTarget, } if config.LightBandwidthIn > 0 { ct.inSizeFactor = utilTarget / float64(config.LightBandwidthIn) @@ -204,8 +212,15 @@ func (ct *costTracker) makeCostList(globalFactor float64) RequestCostList { return list } -type gfUpdate struct { - avgTimeCost, servingTime float64 +// reqInfo contains the estimated time cost and the actual request serving time +// which acts as a feed source to update factor maintained by costTracker. +type reqInfo struct { + // avgTimeCost is the estimated time cost corresponding to maxCostTable. + avgTimeCost float64 + + // servingTime is the CPU time corresponding to the actual processing of + // the request. + servingTime float64 } // gfLoop starts an event loop which updates the global cost factor which is @@ -218,43 +233,48 @@ type gfUpdate struct { // total allowed serving time per second but nominated in cost units, should // also be scaled with the cost factor and is also updated by this loop. func (ct *costTracker) gfLoop() { - var gfLog, recentTime, recentAvg float64 - lastUpdate := mclock.Now() - expUpdate := lastUpdate + var ( + factor, totalRecharge float64 + gfLog, recentTime, recentAvg float64 + lastUpdate, expUpdate = mclock.Now(), mclock.Now() + ) + + // Load historical cost factor statistics from the database. data, _ := ct.db.Get([]byte(gfDbKey)) if len(data) == 8 { gfLog = math.Float64frombits(binary.BigEndian.Uint64(data[:])) } - gf := math.Exp(gfLog) - ct.gf = gf - totalRecharge := ct.utilTarget * gf - ct.gfUpdateCh = make(chan gfUpdate, 100) - threshold := gfUsageThreshold * float64(gfUsageTC) * ct.utilTarget / 1000000 + ct.factor = math.Exp(gfLog) + factor, totalRecharge = ct.factor, ct.utilTarget*ct.factor + + // In order to perform factor data statistics under the high request pressure, + // we only adjust factor when recent factor usage beyond the threshold. + threshold := gfUsageThreshold * float64(gfUsageTC) * ct.utilTarget / flowcontrol.FixedPointMultiplier go func() { saveCostFactor := func() { var data [8]byte binary.BigEndian.PutUint64(data[:], math.Float64bits(gfLog)) ct.db.Put([]byte(gfDbKey), data[:]) - log.Debug("global cost factor saved", "value", gf) + log.Debug("global cost factor saved", "value", factor) } saveTicker := time.NewTicker(time.Minute * 10) for { select { - case r := <-ct.gfUpdateCh: + case r := <-ct.reqInfoCh: + requestServedMeter.Mark(int64(r.servingTime)) + requestEstimatedMeter.Mark(int64(r.avgTimeCost / factor)) + requestServedTimer.Update(time.Duration(r.servingTime)) + relativeCostHistogram.Update(int64(r.avgTimeCost / factor / r.servingTime)) + now := mclock.Now() - if ct.logRelCost != nil && r.avgTimeCost > 1e-20 { - ct.logRelCost.Update(r.servingTime * gf / r.avgTimeCost) - } - if r.servingTime > 1000000000 { - ct.logger.Event(fmt.Sprintf("Very long servingTime = %f avgTimeCost = %f costFactor = %f", r.servingTime, r.avgTimeCost, gf)) - } dt := float64(now - expUpdate) expUpdate = now exp := math.Exp(-dt / float64(gfUsageTC)) - // calculate gf correction until now, based on previous values + + // calculate factor correction until now, based on previous values var gfCorr float64 max := recentTime if recentAvg > max { @@ -268,27 +288,28 @@ func (ct *costTracker) gfLoop() { } else { gfCorr = math.Log(max/threshold) * float64(gfUsageTC) } - // calculate log(gf) correction with the right direction and time constant + // calculate log(factor) correction with the right direction and time constant if recentTime > recentAvg { - // drop gf if actual serving times are larger than average estimates + // drop factor if actual serving times are larger than average estimates gfCorr /= -float64(gfDropTC) } else { - // raise gf if actual serving times are smaller than average estimates + // raise factor if actual serving times are smaller than average estimates gfCorr /= float64(gfRaiseTC) } } // update recent cost values with current request recentTime = recentTime*exp + r.servingTime - recentAvg = recentAvg*exp + r.avgTimeCost/gf + recentAvg = recentAvg*exp + r.avgTimeCost/factor if gfCorr != 0 { + // Apply the correction to factor gfLog += gfCorr - gf = math.Exp(gfLog) + factor = math.Exp(gfLog) + // Notify outside modules the new factor and totalRecharge. if time.Duration(now-lastUpdate) > time.Second { - totalRecharge = ct.utilTarget * gf - lastUpdate = now + totalRecharge, lastUpdate = ct.utilTarget*factor, now ct.gfLock.Lock() - ct.gf = gf + ct.factor = factor ch := ct.totalRechargeCh ct.gfLock.Unlock() if ch != nil { @@ -297,12 +318,12 @@ func (ct *costTracker) gfLoop() { default: } } - log.Debug("global cost factor updated", "gf", gf) + log.Debug("global cost factor updated", "factor", factor) } } - ct.logRecentTime.Update(recentTime) - ct.logRecentAvg.Update(recentAvg) - ct.logTotalRecharge.Update(totalRecharge) + recentServedGauge.Update(int64(recentTime)) + recentEstimatedGauge.Update(int64(recentAvg)) + totalRechargeGauge.Update(int64(totalRecharge)) case <-saveTicker.C: saveCostFactor() @@ -321,7 +342,7 @@ func (ct *costTracker) globalFactor() float64 { ct.gfLock.RLock() defer ct.gfLock.RUnlock() - return ct.gf + return ct.factor } // totalRecharge returns the current total recharge parameter which is used by @@ -330,7 +351,7 @@ func (ct *costTracker) totalRecharge() uint64 { ct.gfLock.RLock() defer ct.gfLock.RUnlock() - return uint64(ct.gf * ct.utilTarget) + return uint64(ct.factor * ct.utilTarget) } // subscribeTotalRecharge returns all future updates to the total recharge value @@ -340,7 +361,7 @@ func (ct *costTracker) subscribeTotalRecharge(ch chan uint64) uint64 { defer ct.gfLock.Unlock() ct.totalRechargeCh = ch - return uint64(ct.gf * ct.utilTarget) + return uint64(ct.factor * ct.utilTarget) } // updateStats updates the global cost factor and (if enabled) the real cost vs. @@ -349,7 +370,7 @@ func (ct *costTracker) updateStats(code, amount, servingTime, realCost uint64) { avg := reqAvgTimeCost[code] avgTimeCost := avg.baseCost + amount*avg.reqCost select { - case ct.gfUpdateCh <- gfUpdate{float64(avgTimeCost), float64(servingTime)}: + case ct.reqInfoCh <- reqInfo{float64(avgTimeCost), float64(servingTime)}: default: } if makeCostStats { diff --git a/les/csvlogger/csvlogger.go b/les/csvlogger/csvlogger.go deleted file mode 100644 index 9a4093cb9a..0000000000 --- a/les/csvlogger/csvlogger.go +++ /dev/null @@ -1,227 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package csvlogger - -import ( - "fmt" - "os" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common/mclock" - "github.com/ethereum/go-ethereum/log" -) - -// Logger is a metrics/events logger that writes logged values and events into a comma separated file -type Logger struct { - file *os.File - started mclock.AbsTime - channels []*Channel - period time.Duration - stopCh, stopped chan struct{} - storeCh chan string - eventHeader string -} - -// NewLogger creates a new Logger -func NewLogger(fileName string, updatePeriod time.Duration, eventHeader string) *Logger { - if fileName == "" { - return nil - } - f, err := os.Create(fileName) - if err != nil { - log.Error("Error creating log file", "name", fileName, "error", err) - return nil - } - return &Logger{ - file: f, - period: updatePeriod, - stopCh: make(chan struct{}), - storeCh: make(chan string, 1), - eventHeader: eventHeader, - } -} - -// NewChannel creates a new value logger channel that writes values in a single -// column. If the relative change of the value is bigger than the given threshold -// then a new line is added immediately (threshold can also be 0). -func (l *Logger) NewChannel(name string, threshold float64) *Channel { - if l == nil { - return nil - } - c := &Channel{ - logger: l, - name: name, - threshold: threshold, - } - l.channels = append(l.channels, c) - return c -} - -// NewMinMaxChannel creates a new value logger channel that writes the minimum and -// maximum of the tracked value in two columns. It never triggers adding a new line. -// If zeroDefault is true then 0 is written to both min and max columns if no update -// was given during the last period. If it is false then the last update will appear -// in both columns. -func (l *Logger) NewMinMaxChannel(name string, zeroDefault bool) *Channel { - if l == nil { - return nil - } - c := &Channel{ - logger: l, - name: name, - minmax: true, - mmZeroDefault: zeroDefault, - } - l.channels = append(l.channels, c) - return c -} - -func (l *Logger) store(event string) { - s := fmt.Sprintf("%g", float64(mclock.Now()-l.started)/1000000000) - for _, ch := range l.channels { - s += ", " + ch.store() - } - if event != "" { - s += ", " + event - } - l.file.WriteString(s + "\n") -} - -// Start writes the header line and starts the logger -func (l *Logger) Start() { - if l == nil { - return - } - l.started = mclock.Now() - s := "Time" - for _, ch := range l.channels { - s += ", " + ch.header() - } - if l.eventHeader != "" { - s += ", " + l.eventHeader - } - l.file.WriteString(s + "\n") - go func() { - timer := time.NewTimer(l.period) - for { - select { - case <-timer.C: - l.store("") - timer.Reset(l.period) - case event := <-l.storeCh: - l.store(event) - if !timer.Stop() { - <-timer.C - } - timer.Reset(l.period) - case <-l.stopCh: - close(l.stopped) - return - } - } - }() -} - -// Stop stops the logger and closes the file -func (l *Logger) Stop() { - if l == nil { - return - } - l.stopped = make(chan struct{}) - close(l.stopCh) - <-l.stopped - l.file.Close() -} - -// Event immediately adds a new line and adds the given event string in the last column -func (l *Logger) Event(event string) { - if l == nil { - return - } - select { - case l.storeCh <- event: - case <-l.stopCh: - } -} - -// Channel represents a logger channel tracking a single value -type Channel struct { - logger *Logger - lock sync.Mutex - name string - threshold, storeMin, storeMax, lastValue, min, max float64 - minmax, mmSet, mmZeroDefault bool -} - -// Update updates the tracked value -func (lc *Channel) Update(value float64) { - if lc == nil { - return - } - lc.lock.Lock() - defer lc.lock.Unlock() - - lc.lastValue = value - if lc.minmax { - if value > lc.max || !lc.mmSet { - lc.max = value - } - if value < lc.min || !lc.mmSet { - lc.min = value - } - lc.mmSet = true - } else { - if value < lc.storeMin || value > lc.storeMax { - select { - case lc.logger.storeCh <- "": - default: - } - } - } -} - -func (lc *Channel) store() (s string) { - lc.lock.Lock() - defer lc.lock.Unlock() - - if lc.minmax { - s = fmt.Sprintf("%g, %g", lc.min, lc.max) - lc.mmSet = false - if lc.mmZeroDefault { - lc.min = 0 - } else { - lc.min = lc.lastValue - } - lc.max = lc.min - } else { - s = fmt.Sprintf("%g", lc.lastValue) - lc.storeMin = lc.lastValue * (1 - lc.threshold) - lc.storeMax = lc.lastValue * (1 + lc.threshold) - if lc.lastValue < 0 { - lc.storeMin, lc.storeMax = lc.storeMax, lc.storeMin - } - } - return -} - -func (lc *Channel) header() string { - if lc.minmax { - return lc.name + " (min), " + lc.name + " (max)" - } - return lc.name -} diff --git a/les/freeclient.go b/les/freeclient.go index f434ea0b95..934b881537 100644 --- a/les/freeclient.go +++ b/les/freeclient.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/les/csvlogger" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) @@ -53,8 +52,7 @@ type freeClientPool struct { connectedLimit, totalLimit int freeClientCap uint64 - logger *csvlogger.Logger - logTotalFreeConn *csvlogger.Channel + connectedCap uint64 addressMap map[string]*freeClientPoolEntry connPool, disconnPool *prque.Prque @@ -69,18 +67,16 @@ const ( ) // newFreeClientPool creates a new free client pool -func newFreeClientPool(db ethdb.Database, freeClientCap uint64, totalLimit int, clock mclock.Clock, removePeer func(string), metricsLogger, eventLogger *csvlogger.Logger) *freeClientPool { +func newFreeClientPool(db ethdb.Database, freeClientCap uint64, totalLimit int, clock mclock.Clock, removePeer func(string)) *freeClientPool { pool := &freeClientPool{ - db: db, - clock: clock, - addressMap: make(map[string]*freeClientPoolEntry), - connPool: prque.New(poolSetIndex), - disconnPool: prque.New(poolSetIndex), - freeClientCap: freeClientCap, - totalLimit: totalLimit, - logger: eventLogger, - logTotalFreeConn: metricsLogger.NewChannel("totalFreeConn", 0), - removePeer: removePeer, + db: db, + clock: clock, + addressMap: make(map[string]*freeClientPoolEntry), + connPool: prque.New(poolSetIndex), + disconnPool: prque.New(poolSetIndex), + freeClientCap: freeClientCap, + totalLimit: totalLimit, + removePeer: removePeer, } pool.loadFromDb() return pool @@ -126,10 +122,7 @@ func (f *freeClientPool) connect(address, id string) bool { if f.closed { return false } - - f.logger.Event("freeClientPool: connecting from " + address + ", " + id) if f.connectedLimit == 0 { - f.logger.Event("freeClientPool: rejected, " + id) log.Debug("Client rejected", "address", address) return false } @@ -141,7 +134,6 @@ func (f *freeClientPool) connect(address, id string) bool { f.addressMap[address] = e } else { if e.connected { - f.logger.Event("freeClientPool: already connected, " + id) log.Debug("Client already connected", "address", address) return false } @@ -154,12 +146,13 @@ func (f *freeClientPool) connect(address, id string) bool { if e.linUsage+int64(connectedBias)-i.linUsage < 0 { // kick it out and accept the new client f.dropClient(i, now) - f.logger.Event("freeClientPool: kicked out, " + i.id) + clientKickedMeter.Mark(1) + f.connectedCap -= f.freeClientCap } else { // keep the old client and reject the new one f.connPool.Push(i, i.linUsage) - f.logger.Event("freeClientPool: rejected, " + id) log.Debug("Client rejected", "address", address) + clientRejectedMeter.Mark(1) return false } } @@ -167,11 +160,12 @@ func (f *freeClientPool) connect(address, id string) bool { e.connected = true e.id = id f.connPool.Push(e, e.linUsage) - f.logTotalFreeConn.Update(float64(uint64(f.connPool.Size()) * f.freeClientCap)) if f.connPool.Size()+f.disconnPool.Size() > f.totalLimit { f.disconnPool.Pop() } - f.logger.Event("freeClientPool: accepted, " + id) + f.connectedCap += f.freeClientCap + totalConnectedGauge.Update(int64(f.connectedCap)) + clientConnectedMeter.Mark(1) log.Debug("Client accepted", "address", address) return true } @@ -203,13 +197,12 @@ func (f *freeClientPool) disconnect(address string) { log.Debug("Client already disconnected", "address", address) return } - f.connPool.Remove(e.index) - f.logTotalFreeConn.Update(float64(uint64(f.connPool.Size()) * f.freeClientCap)) f.calcLogUsage(e, now) e.connected = false f.disconnPool.Push(e, -e.logUsage) - f.logger.Event("freeClientPool: disconnected, " + e.id) + f.connectedCap -= f.freeClientCap + totalConnectedGauge.Update(int64(f.connectedCap)) log.Debug("Client disconnected", "address", address) } @@ -227,15 +220,15 @@ func (f *freeClientPool) setLimits(count int, totalCap uint64) { for f.connPool.Size() > f.connectedLimit { i := f.connPool.PopItem().(*freeClientPoolEntry) f.dropClient(i, now) - f.logger.Event("freeClientPool: setLimits kicked out, " + i.id) + f.connectedCap -= f.freeClientCap } + totalConnectedGauge.Update(int64(f.connectedCap)) } // dropClient disconnects a client and also moves it from the connected to the // disconnected pool func (f *freeClientPool) dropClient(i *freeClientPoolEntry, now mclock.AbsTime) { f.connPool.Remove(i.index) - f.logTotalFreeConn.Update(float64(uint64(f.connPool.Size()) * f.freeClientCap)) f.calcLogUsage(i, now) i.connected = false f.disconnPool.Push(i, -i.logUsage) diff --git a/les/freeclient_test.go b/les/freeclient_test.go index 5a58a6c1c7..1918222641 100644 --- a/les/freeclient_test.go +++ b/les/freeclient_test.go @@ -61,7 +61,7 @@ func testFreeClientPool(t *testing.T, connLimit, clientCount int) { } disconnCh <- i } - pool = newFreeClientPool(db, 1, 10000, &clock, disconnFn, nil, nil) + pool = newFreeClientPool(db, 1, 10000, &clock, disconnFn) ) pool.setLimits(connLimit, uint64(connLimit)) @@ -130,7 +130,7 @@ func testFreeClientPool(t *testing.T, connLimit, clientCount int) { // close and restart pool pool.stop() - pool = newFreeClientPool(db, 1, 10000, &clock, disconnFn, nil, nil) + pool = newFreeClientPool(db, 1, 10000, &clock, disconnFn) pool.setLimits(connLimit, uint64(connLimit)) // try connecting all known peers (connLimit should be filled up) diff --git a/les/handler.go b/les/handler.go index c902db65ac..d9d07f014d 100644 --- a/les/handler.go +++ b/les/handler.go @@ -35,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/les/csvlogger" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" @@ -124,7 +123,6 @@ type ProtocolManager struct { wg *sync.WaitGroup eventMux *event.TypeMux - logger *csvlogger.Logger // Callbacks synced func() bool @@ -262,11 +260,12 @@ func (pm *ProtocolManager) handle(p *peer) error { // Ignore maxPeers if this is a trusted peer // In server mode we try to check into the client pool after handshake if pm.client && pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted { - pm.logger.Event("Rejected (too many peers), " + p.id) + clientRejectedMeter.Mark(1) return p2p.DiscTooManyPeers } // Reject light clients if server is not synced. if !pm.client && !pm.synced() { + clientRejectedMeter.Mark(1) return p2p.DiscRequested } p.Log().Debug("Light Ethereum peer connected", "name", p.Name()) @@ -281,7 +280,7 @@ func (pm *ProtocolManager) handle(p *peer) error { ) if err := p.Handshake(td, hash, number, genesis.Hash(), pm.server); err != nil { p.Log().Debug("Light Ethereum handshake failed", "err", err) - pm.logger.Event("Handshake error: " + err.Error() + ", " + p.id) + clientErrorMeter.Mark(1) return err } if p.fcClient != nil { @@ -294,14 +293,14 @@ func (pm *ProtocolManager) handle(p *peer) error { // Register the peer locally if err := pm.peers.Register(p); err != nil { + clientErrorMeter.Mark(1) p.Log().Error("Light Ethereum peer registration failed", "err", err) - pm.logger.Event("Peer registration error: " + err.Error() + ", " + p.id) return err } - pm.logger.Event("Connection established, " + p.id) + connectedAt := time.Now() defer func() { - pm.logger.Event("Closed connection, " + p.id) pm.removePeer(p.id) + connectionTimer.UpdateSince(connectedAt) }() // Register the peer in the downloader. If the downloader considers it banned, we disconnect @@ -317,11 +316,9 @@ func (pm *ProtocolManager) handle(p *peer) error { pm.serverPool.registered(p.poolEntry) } } - // main loop. handle incoming messages. for { if err := pm.handleMsg(p); err != nil { - pm.logger.Event("Message handling error: " + err.Error() + ", " + p.id) p.Log().Debug("Light Ethereum message handling failed", "err", err) if p.fcServer != nil { p.fcServer.DumpLogs() diff --git a/les/helper_test.go b/les/helper_test.go index 035865b08d..fd5236a99e 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -231,7 +231,7 @@ func newTestProtocolManager(lightSync bool, blocks int, odr *LesOdr, indexers [] if !lightSync { srv := &LesServer{lesCommons: lesCommons{protocolManager: pm, chainDb: db}} pm.server = srv - pm.servingQueue = newServingQueue(int64(time.Millisecond*10), 1, nil) + pm.servingQueue = newServingQueue(int64(time.Millisecond*10), 1) pm.servingQueue.setThreads(4) srv.defParams = flowcontrol.ServerParams{ diff --git a/les/metrics.go b/les/metrics.go index c282a62a1a..4c6737a4e8 100644 --- a/les/metrics.go +++ b/les/metrics.go @@ -22,46 +22,31 @@ import ( ) var ( - /* propTxnInPacketsMeter = metrics.NewMeter("eth/prop/txns/in/packets") - propTxnInTrafficMeter = metrics.NewMeter("eth/prop/txns/in/traffic") - propTxnOutPacketsMeter = metrics.NewMeter("eth/prop/txns/out/packets") - propTxnOutTrafficMeter = metrics.NewMeter("eth/prop/txns/out/traffic") - propHashInPacketsMeter = metrics.NewMeter("eth/prop/hashes/in/packets") - propHashInTrafficMeter = metrics.NewMeter("eth/prop/hashes/in/traffic") - propHashOutPacketsMeter = metrics.NewMeter("eth/prop/hashes/out/packets") - propHashOutTrafficMeter = metrics.NewMeter("eth/prop/hashes/out/traffic") - propBlockInPacketsMeter = metrics.NewMeter("eth/prop/blocks/in/packets") - propBlockInTrafficMeter = metrics.NewMeter("eth/prop/blocks/in/traffic") - propBlockOutPacketsMeter = metrics.NewMeter("eth/prop/blocks/out/packets") - propBlockOutTrafficMeter = metrics.NewMeter("eth/prop/blocks/out/traffic") - reqHashInPacketsMeter = metrics.NewMeter("eth/req/hashes/in/packets") - reqHashInTrafficMeter = metrics.NewMeter("eth/req/hashes/in/traffic") - reqHashOutPacketsMeter = metrics.NewMeter("eth/req/hashes/out/packets") - reqHashOutTrafficMeter = metrics.NewMeter("eth/req/hashes/out/traffic") - reqBlockInPacketsMeter = metrics.NewMeter("eth/req/blocks/in/packets") - reqBlockInTrafficMeter = metrics.NewMeter("eth/req/blocks/in/traffic") - reqBlockOutPacketsMeter = metrics.NewMeter("eth/req/blocks/out/packets") - reqBlockOutTrafficMeter = metrics.NewMeter("eth/req/blocks/out/traffic") - reqHeaderInPacketsMeter = metrics.NewMeter("eth/req/headers/in/packets") - reqHeaderInTrafficMeter = metrics.NewMeter("eth/req/headers/in/traffic") - reqHeaderOutPacketsMeter = metrics.NewMeter("eth/req/headers/out/packets") - reqHeaderOutTrafficMeter = metrics.NewMeter("eth/req/headers/out/traffic") - reqBodyInPacketsMeter = metrics.NewMeter("eth/req/bodies/in/packets") - reqBodyInTrafficMeter = metrics.NewMeter("eth/req/bodies/in/traffic") - reqBodyOutPacketsMeter = metrics.NewMeter("eth/req/bodies/out/packets") - reqBodyOutTrafficMeter = metrics.NewMeter("eth/req/bodies/out/traffic") - reqStateInPacketsMeter = metrics.NewMeter("eth/req/states/in/packets") - reqStateInTrafficMeter = metrics.NewMeter("eth/req/states/in/traffic") - reqStateOutPacketsMeter = metrics.NewMeter("eth/req/states/out/packets") - reqStateOutTrafficMeter = metrics.NewMeter("eth/req/states/out/traffic") - reqReceiptInPacketsMeter = metrics.NewMeter("eth/req/receipts/in/packets") - reqReceiptInTrafficMeter = metrics.NewMeter("eth/req/receipts/in/traffic") - reqReceiptOutPacketsMeter = metrics.NewMeter("eth/req/receipts/out/packets") - reqReceiptOutTrafficMeter = metrics.NewMeter("eth/req/receipts/out/traffic")*/ miscInPacketsMeter = metrics.NewRegisteredMeter("les/misc/in/packets", nil) miscInTrafficMeter = metrics.NewRegisteredMeter("les/misc/in/traffic", nil) miscOutPacketsMeter = metrics.NewRegisteredMeter("les/misc/out/packets", nil) miscOutTrafficMeter = metrics.NewRegisteredMeter("les/misc/out/traffic", nil) + + connectionTimer = metrics.NewRegisteredTimer("les/connectionTime", nil) + + totalConnectedGauge = metrics.NewRegisteredGauge("les/server/totalConnected", nil) + totalCapacityGauge = metrics.NewRegisteredGauge("les/server/totalCapacity", nil) + totalRechargeGauge = metrics.NewRegisteredGauge("les/server/totalRecharge", nil) + blockProcessingTimer = metrics.NewRegisteredTimer("les/server/blockProcessingTime", nil) + requestServedTimer = metrics.NewRegisteredTimer("les/server/requestServed", nil) + requestServedMeter = metrics.NewRegisteredMeter("les/server/totalRequestServed", nil) + requestEstimatedMeter = metrics.NewRegisteredMeter("les/server/totalRequestEstimated", nil) + relativeCostHistogram = metrics.NewRegisteredHistogram("les/server/relativeCost", nil, metrics.NewExpDecaySample(1028, 0.015)) + recentServedGauge = metrics.NewRegisteredGauge("les/server/recentRequestServed", nil) + recentEstimatedGauge = metrics.NewRegisteredGauge("les/server/recentRequestEstimated", nil) + sqServedGauge = metrics.NewRegisteredGauge("les/server/servingQueue/served", nil) + sqQueuedGauge = metrics.NewRegisteredGauge("les/server/servingQueue/queued", nil) + clientConnectedMeter = metrics.NewRegisteredMeter("les/server/clientEvent/connected", nil) + clientRejectedMeter = metrics.NewRegisteredMeter("les/server/clientEvent/rejected", nil) + clientKickedMeter = metrics.NewRegisteredMeter("les/server/clientEvent/kicked", nil) + // clientDisconnectedMeter = metrics.NewRegisteredMeter("les/server/clientEvent/disconnected", nil) + clientFreezeMeter = metrics.NewRegisteredMeter("les/server/clientEvent/freeze", nil) + clientErrorMeter = metrics.NewRegisteredMeter("les/server/clientEvent/error", nil) ) // meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of diff --git a/les/server.go b/les/server.go index 08d9734165..86570aa54d 100644 --- a/les/server.go +++ b/les/server.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/les/csvlogger" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" @@ -40,15 +39,6 @@ import ( const bufLimitRatio = 6000 // fixed bufLimit/MRR ratio -const ( - logFileName = "" // csv log file name (disabled if empty) - logClientPoolMetrics = true // log client pool metrics - logClientPoolEvents = false // detailed client pool event logging - logRequestServing = true // log request serving metrics and events - logBlockProcEvents = true // log block processing events - logProtocolHandler = true // log protocol handler events -) - type LesServer struct { lesCommons @@ -62,26 +52,15 @@ type LesServer struct { privateKey *ecdsa.PrivateKey quitSync chan struct{} onlyAnnounce bool - csvLogger *csvlogger.Logger - logTotalCap *csvlogger.Channel thcNormal, thcBlockProcessing int // serving thread count for normal operation and block processing mode maxPeers int minCapacity, freeClientCap uint64 freeClientPool *freeClientPool - priorityClientPool *priorityClientPool } func NewLesServer(e *eth.Ethereum, config *eth.Config) (*LesServer, error) { - var csvLogger *csvlogger.Logger - if logFileName != "" { - csvLogger = csvlogger.NewLogger(logFileName, time.Second*10, "event, peerId") - } - requestLogger := csvLogger - if !logRequestServing { - requestLogger = nil - } lesTopics := make([]discv5.Topic, len(AdvertiseProtocolVersions)) for i, pv := range AdvertiseProtocolVersions { lesTopics[i] = lesTopic(e.BlockChain().Genesis().Hash(), pv) @@ -99,10 +78,8 @@ func NewLesServer(e *eth.Ethereum, config *eth.Config) (*LesServer, error) { quitSync: quitSync, lesTopics: lesTopics, onlyAnnounce: config.OnlyAnnounce, - csvLogger: csvLogger, - logTotalCap: requestLogger.NewChannel("totalCapacity", 0.01), } - srv.costTracker, srv.minCapacity = newCostTracker(e.ChainDb(), config, requestLogger) + srv.costTracker, srv.minCapacity = newCostTracker(e.ChainDb(), config) logger := log.New() srv.thcNormal = config.LightServ * 4 / 100 @@ -131,10 +108,7 @@ func NewLesServer(e *eth.Ethereum, config *eth.Config) (*LesServer, error) { return nil, err } srv.protocolManager = pm - if logProtocolHandler { - pm.logger = csvLogger - } - pm.servingQueue = newServingQueue(int64(time.Millisecond*10), float64(config.LightServ)/100, requestLogger) + pm.servingQueue = newServingQueue(int64(time.Millisecond*10), float64(config.LightServ)/100) pm.server = srv return srv, nil @@ -142,12 +116,6 @@ func NewLesServer(e *eth.Ethereum, config *eth.Config) (*LesServer, error) { func (s *LesServer) APIs() []rpc.API { return []rpc.API{ - { - Namespace: "les", - Version: "1.0", - Service: NewPrivateLightServerAPI(s), - Public: false, - }, { Namespace: "les", Version: "1.0", @@ -163,11 +131,10 @@ func (s *LesServer) APIs() []rpc.API { func (s *LesServer) startEventLoop() { s.protocolManager.wg.Add(1) - blockProcLogger := s.csvLogger - if !logBlockProcEvents { - blockProcLogger = nil - } - var processing, procLast bool + var ( + processing, procLast bool + procStarted time.Time + ) blockProcFeed := make(chan bool, 100) s.protocolManager.blockchain.(*core.BlockChain).SubscribeBlockProcessingEvent(blockProcFeed) totalRechargeCh := make(chan uint64, 100) @@ -176,13 +143,13 @@ func (s *LesServer) startEventLoop() { updateRecharge := func() { if processing { if !procLast { - blockProcLogger.Event("block processing started") + procStarted = time.Now() } s.protocolManager.servingQueue.setThreads(s.thcBlockProcessing) s.fcManager.SetRechargeCurve(flowcontrol.PieceWiseLinear{{0, 0}, {totalRecharge, totalRecharge}}) } else { if procLast { - blockProcLogger.Event("block processing finished") + blockProcessingTimer.UpdateSince(procStarted) } s.protocolManager.servingQueue.setThreads(s.thcNormal) s.fcManager.SetRechargeCurve(flowcontrol.PieceWiseLinear{{0, 0}, {totalRecharge / 16, totalRecharge / 2}, {totalRecharge / 2, totalRecharge / 2}, {totalRecharge, totalRecharge}}) @@ -191,7 +158,7 @@ func (s *LesServer) startEventLoop() { } updateRecharge() totalCapacity := s.fcManager.SubscribeTotalCapacity(totalCapacityCh) - s.priorityClientPool.setLimits(s.maxPeers, totalCapacity) + s.freeClientPool.setLimits(s.maxPeers, totalCapacity) var maxFreePeers uint64 go func() { @@ -202,13 +169,13 @@ func (s *LesServer) startEventLoop() { case totalRecharge = <-totalRechargeCh: updateRecharge() case totalCapacity = <-totalCapacityCh: - s.logTotalCap.Update(float64(totalCapacity)) + totalCapacityGauge.Update(int64(totalCapacity)) newFreePeers := totalCapacity / s.freeClientCap if newFreePeers < maxFreePeers && newFreePeers < uint64(s.maxPeers) { log.Warn("Reduced total capacity", "maxFreePeers", newFreePeers) } maxFreePeers = newFreePeers - s.priorityClientPool.setLimits(s.maxPeers, totalCapacity) + s.freeClientPool.setLimits(s.maxPeers, totalCapacity) case <-s.protocolManager.quitSync: s.protocolManager.wg.Done() return @@ -243,19 +210,9 @@ func (s *LesServer) Start(srvr *p2p.Server) { maxCapacity = totalRecharge } s.fcManager.SetCapacityLimits(s.freeClientCap, maxCapacity, s.freeClientCap*2) - poolMetricsLogger := s.csvLogger - if !logClientPoolMetrics { - poolMetricsLogger = nil - } - poolEventLogger := s.csvLogger - if !logClientPoolEvents { - poolEventLogger = nil - } - s.freeClientPool = newFreeClientPool(s.chainDb, s.freeClientCap, 10000, mclock.System{}, func(id string) { go s.protocolManager.removePeer(id) }, poolMetricsLogger, poolEventLogger) - s.priorityClientPool = newPriorityClientPool(s.freeClientCap, s.protocolManager.peers, s.freeClientPool, poolMetricsLogger, poolEventLogger) + s.freeClientPool = newFreeClientPool(s.chainDb, s.freeClientCap, 10000, mclock.System{}, func(id string) { go s.protocolManager.removePeer(id) }) + s.protocolManager.peers.notify(s.freeClientPool) - s.protocolManager.peers.notify(s.priorityClientPool) - s.csvLogger.Start() s.startEventLoop() s.protocolManager.Start(s.config.LightPeers) if srvr.DiscV5 != nil { @@ -296,7 +253,6 @@ func (s *LesServer) Stop() { s.freeClientPool.stop() s.costTracker.stop() s.protocolManager.Stop() - s.csvLogger.Stop() } // todo(rjl493456442) separate client and server implementation. diff --git a/les/servingqueue.go b/les/servingqueue.go index 26656ec018..a9e8369fe1 100644 --- a/les/servingqueue.go +++ b/les/servingqueue.go @@ -17,14 +17,12 @@ package les import ( - "fmt" "sort" "sync" "sync/atomic" "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/common/prque" - "github.com/ethereum/go-ethereum/les/csvlogger" ) // servingQueue allows running tasks in a limited number of threads and puts the @@ -44,10 +42,6 @@ type servingQueue struct { queue *prque.Prque // priority queue for waiting or suspended tasks best *servingTask // the highest priority task (not included in the queue) suspendBias int64 // priority bias against suspending an already running task - - logger *csvlogger.Logger - logRecentTime *csvlogger.Channel - logQueuedTime *csvlogger.Channel } // servingTask represents a request serving task. Tasks can be implemented to @@ -127,7 +121,7 @@ func (t *servingTask) waitOrStop() bool { } // newServingQueue returns a new servingQueue -func newServingQueue(suspendBias int64, utilTarget float64, logger *csvlogger.Logger) *servingQueue { +func newServingQueue(suspendBias int64, utilTarget float64) *servingQueue { sq := &servingQueue{ queue: prque.New(nil), suspendBias: suspendBias, @@ -140,9 +134,6 @@ func newServingQueue(suspendBias int64, utilTarget float64, logger *csvlogger.Lo burstDropLimit: uint64(utilTarget * bufLimitRatio * 1000000), burstDecRate: utilTarget, lastUpdate: mclock.Now(), - logger: logger, - logRecentTime: logger.NewMinMaxChannel("recentTime", false), - logQueuedTime: logger.NewMinMaxChannel("queuedTime", false), } sq.wg.Add(2) go sq.queueLoop() @@ -246,16 +237,13 @@ func (sq *servingQueue) freezePeers() { } sort.Sort(peerList) drop := true - sq.logger.Event("freezing peers") for _, tasks := range peerList { if drop { tasks.peer.freezeClient() tasks.peer.fcClient.Freeze() sq.queuedTime -= tasks.sumTime - if sq.logQueuedTime != nil { - sq.logQueuedTime.Update(float64(sq.queuedTime) / 1000) - } - sq.logger.Event(fmt.Sprintf("frozen peer sumTime=%d, %v", tasks.sumTime, tasks.peer.id)) + sqQueuedGauge.Update(int64(sq.queuedTime)) + clientFreezeMeter.Mark(1) drop = sq.recentTime+sq.queuedTime > sq.burstDropLimit for _, task := range tasks.list { task.tokenCh <- nil @@ -299,10 +287,8 @@ func (sq *servingQueue) addTask(task *servingTask) { } sq.updateRecentTime() sq.queuedTime += task.expTime - if sq.logQueuedTime != nil { - sq.logRecentTime.Update(float64(sq.recentTime) / 1000) - sq.logQueuedTime.Update(float64(sq.queuedTime) / 1000) - } + sqServedGauge.Update(int64(sq.recentTime)) + sqQueuedGauge.Update(int64(sq.queuedTime)) if sq.recentTime+sq.queuedTime > sq.burstLimit { sq.freezePeers() } @@ -322,10 +308,8 @@ func (sq *servingQueue) queueLoop() { sq.updateRecentTime() sq.queuedTime -= expTime sq.recentTime += expTime - if sq.logQueuedTime != nil { - sq.logRecentTime.Update(float64(sq.recentTime) / 1000) - sq.logQueuedTime.Update(float64(sq.queuedTime) / 1000) - } + sqServedGauge.Update(int64(sq.recentTime)) + sqQueuedGauge.Update(int64(sq.queuedTime)) if sq.queue.Size() == 0 { sq.best = nil } else { diff --git a/les/sync_test.go b/les/sync_test.go index 634be8e6d0..f5d1ad5bc5 100644 --- a/les/sync_test.go +++ b/les/sync_test.go @@ -22,6 +22,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/light" @@ -82,7 +83,7 @@ func testCheckpointSyncing(t *testing.T, protocol int, syncMode int) { data := append([]byte{0x19, 0x00}, append(registrarAddr.Bytes(), append([]byte{0, 0, 0, 0, 0, 0, 0, 0}, cp.Hash().Bytes()...)...)...) sig, _ := crypto.Sign(crypto.Keccak256(data), signerKey) sig[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper - if _, err := server.pm.reg.contract.RegisterCheckpoint(signerKey, cp.SectionIndex, cp.Hash().Bytes(), new(big.Int).Sub(header.Number, big.NewInt(1)), header.ParentHash, [][]byte{sig}); err != nil { + if _, err := server.pm.reg.contract.RegisterCheckpoint(bind.NewKeyedTransactor(signerKey), cp.SectionIndex, cp.Hash().Bytes(), new(big.Int).Sub(header.Number, big.NewInt(1)), header.ParentHash, [][]byte{sig}); err != nil { t.Error("register checkpoint failed", err) } server.backend.Commit() diff --git a/signer/core/cliui.go b/signer/core/cliui.go index 46a13f1e4c..381c40be3b 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -171,7 +171,7 @@ func (ui *CommandlineUI) ApproveSignData(request *SignDataRequest) (SignDataResp fmt.Printf("Account: %s\n", request.Address.String()) fmt.Printf("messages:\n") for _, nvt := range request.Messages { - fmt.Printf("%v\n", nvt.Pprint(1)) + fmt.Printf("\u00a0\u00a0%v\n", strings.TrimSpace(nvt.Pprint(1))) } fmt.Printf("raw data: \n%q\n", request.Rawdata) fmt.Printf("data hash: %v\n", request.Hash) diff --git a/vendor/github.com/karalabe/usb/usb_disabled.go b/vendor/github.com/karalabe/usb/usb_disabled.go index f7f42a4cfe..77c9007611 100644 --- a/vendor/github.com/karalabe/usb/usb_disabled.go +++ b/vendor/github.com/karalabe/usb/usb_disabled.go @@ -44,3 +44,8 @@ func EnumerateRaw(vendorID uint16, productID uint16) ([]DeviceInfo, error) { func EnumerateHid(vendorID uint16, productID uint16) ([]DeviceInfo, error) { return nil, nil } + +// Open connects to a previsouly discovered USB device. +func (info DeviceInfo) Open() (Device, error) { + return nil, ErrUnsupportedPlatform +} diff --git a/vendor/vendor.json b/vendor/vendor.json index bbf0a01156..59796181b7 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -243,10 +243,10 @@ "revisionTime": "2017-04-30T22:20:11Z" }, { - "checksumSHA1": "TU/WaqL7fYPDovmGVRSo8btD4ZM=", + "checksumSHA1": "X7ZY5gt+qBd/lafKNbPbouL819w=", "path": "github.com/karalabe/usb", - "revision": "4d6ba34a841453237dba721eeec1af0ef9a7a890", - "revisionTime": "2019-06-13T07:02:55Z", + "revision": "6a7de9d893feb2324aaef49331e923ce279c7973", + "revisionTime": "2019-07-03T09:51:11Z", "tree": true }, {