diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c03fa06c72..e2e25180a1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,4 +10,7 @@ les/ @zsfelfoldi light/ @zsfelfoldi mobile/ @karalabe p2p/ @fjl @zsfelfoldi +p2p/simulations @zelig @nonsense @janos @justelad +p2p/protocols @zelig @nonsense @janos @justelad +p2p/testing @zelig @nonsense @janos @justelad whisper/ @gballet @gluk256 diff --git a/.travis.yml b/.travis.yml index b412770854..f08083d3c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -156,7 +156,7 @@ matrix: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://storage.googleapis.com/golang/go1.11.4.linux-amd64.tar.gz | tar -xz + - curl https://storage.googleapis.com/golang/go1.11.5.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index 1b0bb00496..ccc6a65932 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -74,7 +74,7 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value { func set(dst, src reflect.Value) error { dstType, srcType := dst.Type(), src.Type() switch { - case dstType.Kind() == reflect.Interface: + case dstType.Kind() == reflect.Interface && dst.Elem().IsValid(): return set(dst.Elem(), src) case dstType.Kind() == reflect.Ptr && dstType.Elem() != derefbigT: return set(dst.Elem(), src) diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index ff88be3d30..fa8a69d05c 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -512,6 +512,11 @@ func TestMethodMultiReturn(t *testing.T) { Int *big.Int } + newInterfaceSlice := func(len int) interface{} { + slice := make([]interface{}, len) + return &slice + } + abi, data, expected := methodMultiReturn(require.New(t)) bigint := new(big.Int) var testCases = []struct { @@ -539,6 +544,16 @@ func TestMethodMultiReturn(t *testing.T) { &[2]interface{}{&expected.Int, &expected.String}, "", "Can unpack into an array", + }, { + &[2]interface{}{}, + &[2]interface{}{expected.Int, expected.String}, + "", + "Can unpack into interface array", + }, { + newInterfaceSlice(2), + &[]interface{}{expected.Int, expected.String}, + "", + "Can unpack into interface slice", }, { &[]interface{}{new(int), new(int)}, &[]interface{}{&expected.Int, &expected.String}, diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index a9d2e9959e..d3e17aba10 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -28,7 +28,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/accounts/usbwallet/internal/trezor" + "github.com/ethereum/go-ethereum/accounts/usbwallet/trezor" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -41,6 +41,9 @@ import ( // encoded passphrase. var ErrTrezorPINNeeded = errors.New("trezor: pin needed") +// ErrTrezorPassphraseNeeded is returned if opening the trezor requires a passphrase +var ErrTrezorPassphraseNeeded = errors.New("trezor: passphrase needed") + // errTrezorReplyInvalidHeader is the error message returned by a Trezor data exchange // if the device replies with a mismatching header. This usually means the device // is in browser mode. @@ -48,12 +51,13 @@ var errTrezorReplyInvalidHeader = errors.New("trezor: invalid reply header") // trezorDriver implements the communication with a Trezor hardware wallet. type trezorDriver struct { - device io.ReadWriter // USB device connection to communicate through - version [3]uint32 // Current version of the Trezor firmware - label string // Current textual label of the Trezor device - pinwait bool // Flags whether the device is waiting for PIN entry - failure error // Any failure that would make the device unusable - log log.Logger // Contextual logger to tag the trezor with its id + device io.ReadWriter // USB device connection to communicate through + version [3]uint32 // Current version of the Trezor firmware + label string // Current textual label of the Trezor device + pinwait bool // Flags whether the device is waiting for PIN entry + passphrasewait bool // Flags whether the device is waiting for passphrase entry + failure error // Any failure that would make the device unusable + log log.Logger // Contextual logger to tag the trezor with its id } // newTrezorDriver creates a new instance of a Trezor USB protocol driver. @@ -79,7 +83,7 @@ func (w *trezorDriver) Status() (string, error) { } // Open implements usbwallet.driver, attempting to initialize the connection to -// the Trezor hardware wallet. Initializing the Trezor is a two phase operation: +// the Trezor hardware wallet. Initializing the Trezor is a two or three phase operation: // * The first phase is to initialize the connection and read the wallet's // features. This phase is invoked is the provided passphrase is empty. The // device will display the pinpad as a result and will return an appropriate @@ -87,11 +91,13 @@ func (w *trezorDriver) Status() (string, error) { // * The second phase is to unlock access to the Trezor, which is done by the // user actually providing a passphrase mapping a keyboard keypad to the pin // number of the user (shuffled according to the pinpad displayed). +// * If needed the device will ask for passphrase which will require calling +// open again with the actual passphrase (3rd phase) func (w *trezorDriver) Open(device io.ReadWriter, passphrase string) error { w.device, w.failure = device, nil // If phase 1 is requested, init the connection and wait for user callback - if passphrase == "" { + if passphrase == "" && !w.passphrasewait { // If we're already waiting for a PIN entry, insta-return if w.pinwait { return ErrTrezorPINNeeded @@ -104,26 +110,46 @@ func (w *trezorDriver) Open(device io.ReadWriter, passphrase string) error { w.version = [3]uint32{features.GetMajorVersion(), features.GetMinorVersion(), features.GetPatchVersion()} w.label = features.GetLabel() - // Do a manual ping, forcing the device to ask for its PIN + // Do a manual ping, forcing the device to ask for its PIN and Passphrase askPin := true - res, err := w.trezorExchange(&trezor.Ping{PinProtection: &askPin}, new(trezor.PinMatrixRequest), new(trezor.Success)) + askPassphrase := true + res, err := w.trezorExchange(&trezor.Ping{PinProtection: &askPin, PassphraseProtection: &askPassphrase}, new(trezor.PinMatrixRequest), new(trezor.PassphraseRequest), new(trezor.Success)) if err != nil { return err } // Only return the PIN request if the device wasn't unlocked until now - if res == 1 { - return nil // Device responded with trezor.Success + switch res { + case 0: + w.pinwait = true + return ErrTrezorPINNeeded + case 1: + w.pinwait = false + w.passphrasewait = true + return ErrTrezorPassphraseNeeded + case 2: + return nil // responded with trezor.Success } - w.pinwait = true - return ErrTrezorPINNeeded } // Phase 2 requested with actual PIN entry - w.pinwait = false - - if _, err := w.trezorExchange(&trezor.PinMatrixAck{Pin: &passphrase}, new(trezor.Success)); err != nil { - w.failure = err - return err + if w.pinwait { + w.pinwait = false + res, err := w.trezorExchange(&trezor.PinMatrixAck{Pin: &passphrase}, new(trezor.Success), new(trezor.PassphraseRequest)) + if err != nil { + w.failure = err + return err + } + if res == 1 { + w.passphrasewait = true + return ErrTrezorPassphraseNeeded + } + } else if w.passphrasewait { + w.passphrasewait = false + if _, err := w.trezorExchange(&trezor.PassphraseAck{Passphrase: &passphrase}, new(trezor.Success)); err != nil { + w.failure = err + return err + } } + return nil } diff --git a/accounts/usbwallet/internal/trezor/messages.pb.go b/accounts/usbwallet/trezor/messages.pb.go similarity index 100% rename from accounts/usbwallet/internal/trezor/messages.pb.go rename to accounts/usbwallet/trezor/messages.pb.go diff --git a/accounts/usbwallet/internal/trezor/messages.proto b/accounts/usbwallet/trezor/messages.proto similarity index 100% rename from accounts/usbwallet/internal/trezor/messages.proto rename to accounts/usbwallet/trezor/messages.proto diff --git a/accounts/usbwallet/internal/trezor/trezor.go b/accounts/usbwallet/trezor/trezor.go similarity index 100% rename from accounts/usbwallet/internal/trezor/trezor.go rename to accounts/usbwallet/trezor/trezor.go diff --git a/accounts/usbwallet/internal/trezor/types.pb.go b/accounts/usbwallet/trezor/types.pb.go similarity index 100% rename from accounts/usbwallet/internal/trezor/types.pb.go rename to accounts/usbwallet/trezor/types.pb.go diff --git a/accounts/usbwallet/internal/trezor/types.proto b/accounts/usbwallet/trezor/types.proto similarity index 100% rename from accounts/usbwallet/internal/trezor/types.proto rename to accounts/usbwallet/trezor/types.proto diff --git a/appveyor.yml b/appveyor.yml index defad29cd2..9ed4f114e3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.4.windows-%GETH_ARCH%.zip - - 7z x go1.11.4.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.5.windows-%GETH_ARCH%.zip + - 7z x go1.11.5.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/build/ci.go b/build/ci.go index 1bbc944714..d2f4ce4fec 100644 --- a/build/ci.go +++ b/build/ci.go @@ -513,7 +513,7 @@ func doDebianSource(cmdline []string) { for _, distro := range debDistros { meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) pkgdir := stageDebianSource(*workdir, meta) - debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc") + debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d") debuild.Dir = pkgdir build.MustRun(debuild) @@ -523,7 +523,7 @@ func doDebianSource(cmdline []string) { build.MustRunCommand("debsign", changes) } if *upload != "" { - build.MustRunCommand("dput", *upload, changes) + build.MustRunCommand("dput", "--passive", "--no-upload-log", *upload, changes) } } } diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 32f7d63bea..a40b32b600 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -112,12 +112,13 @@ func main() { if !realaddr.IP.IsLoopback() { go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") } - // TODO: react to external IP changes over time. if ext, err := natm.ExternalIP(); err == nil { realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} } } + printNotice(&nodeKey.PublicKey, *realaddr) + if *runv5 { if _, err := discv5.ListenUDP(nodeKey, conn, "", restrictList); err != nil { utils.Fatalf("%v", err) @@ -136,3 +137,13 @@ func main() { select {} } + +func printNotice(nodeKey *ecdsa.PublicKey, addr net.UDPAddr) { + if addr.IP.IsUnspecified() { + addr.IP = net.IP{127, 0, 0, 1} + } + n := enode.NewV4(nodeKey, addr.IP, 0, addr.Port) + fmt.Println(n.String()) + fmt.Println("Note: you're using cmd/bootnode, a developer tool.") + fmt.Println("We recommend using a regular node as bootstrap node for production deployments.") +} diff --git a/cmd/geth/config.go b/cmd/geth/config.go index f1e2811966..f316380cee 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/dashboard" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/graphql" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" @@ -124,6 +125,7 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { } // Apply flags. + utils.SetULC(ctx, &cfg.Eth) utils.SetNodeConfig(ctx, &cfg.Node) stack, err := node.New(&cfg.Node) if err != nil { @@ -176,6 +178,13 @@ func makeFullNode(ctx *cli.Context) *node.Node { utils.RegisterShhService(stack, &cfg.Shh) } + // Configure GraphQL if required + if ctx.GlobalIsSet(utils.GraphQLEnabledFlag.Name) { + if err := graphql.RegisterGraphQLService(stack, cfg.Node.GraphQLEndpoint(), cfg.Node.GraphQLCors, cfg.Node.GraphQLVirtualHosts, cfg.Node.HTTPTimeouts); err != nil { + utils.Fatalf("Failed to register the Ethereum service: %v", err) + } + } + // Add the Ethereum Stats daemon if requested. if cfg.Ethstats.URL != "" { utils.RegisterEthStatsService(stack, cfg.Ethstats.URL) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 97033c692a..77c04ff361 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -38,7 +38,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) const ( @@ -82,6 +82,10 @@ var ( utils.TxPoolAccountQueueFlag, utils.TxPoolGlobalQueueFlag, utils.TxPoolLifetimeFlag, + utils.ULCModeConfigFlag, + utils.OnlyAnnounceModeFlag, + utils.ULCTrustedNodesFlag, + utils.ULCMinTrustedFractionFlag, utils.SyncModeFlag, utils.GCModeFlag, utils.LightServFlag, @@ -141,6 +145,11 @@ var ( utils.RPCEnabledFlag, utils.RPCListenAddrFlag, utils.RPCPortFlag, + utils.GraphQLEnabledFlag, + utils.GraphQLListenAddrFlag, + utils.GraphQLPortFlag, + utils.GraphQLCORSDomainFlag, + utils.GraphQLVirtualHostsFlag, utils.RPCApiFlag, utils.WSEnabledFlag, utils.WSListenAddrFlag, @@ -164,7 +173,7 @@ var ( utils.MetricsInfluxDBDatabaseFlag, utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, - utils.MetricsInfluxDBHostTagFlag, + utils.MetricsInfluxDBTagsFlag, } ) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index fd8ec7cd99..de2a3d0731 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/internal/debug" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) // AppHelpTemplate is the test template for the default, global app help topic. @@ -229,7 +229,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.MetricsInfluxDBDatabaseFlag, utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, - utils.MetricsInfluxDBHostTagFlag, + utils.MetricsInfluxDBTagsFlag, }, }, { diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index c95c81a6dd..ae7675cd9b 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -223,28 +223,29 @@ type parityChainSpec struct { } `json:"engine"` Params struct { - AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` - NetworkID hexutil.Uint64 `json:"networkID"` - ChainID hexutil.Uint64 `json:"chainID"` - MaxCodeSize hexutil.Uint64 `json:"maxCodeSize"` - MaxCodeSizeTransition hexutil.Uint64 `json:"maxCodeSizeTransition"` - EIP98Transition hexutil.Uint64 `json:"eip98Transition"` - EIP150Transition hexutil.Uint64 `json:"eip150Transition"` - EIP160Transition hexutil.Uint64 `json:"eip160Transition"` - EIP161abcTransition hexutil.Uint64 `json:"eip161abcTransition"` - EIP161dTransition hexutil.Uint64 `json:"eip161dTransition"` - EIP155Transition hexutil.Uint64 `json:"eip155Transition"` - EIP140Transition hexutil.Uint64 `json:"eip140Transition"` - EIP211Transition hexutil.Uint64 `json:"eip211Transition"` - EIP214Transition hexutil.Uint64 `json:"eip214Transition"` - EIP658Transition hexutil.Uint64 `json:"eip658Transition"` - EIP145Transition hexutil.Uint64 `json:"eip145Transition"` - EIP1014Transition hexutil.Uint64 `json:"eip1014Transition"` - EIP1052Transition hexutil.Uint64 `json:"eip1052Transition"` - EIP1283Transition hexutil.Uint64 `json:"eip1283Transition"` + AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` + MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` + MinGasLimit hexutil.Uint64 `json:"minGasLimit"` + GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` + NetworkID hexutil.Uint64 `json:"networkID"` + ChainID hexutil.Uint64 `json:"chainID"` + MaxCodeSize hexutil.Uint64 `json:"maxCodeSize"` + MaxCodeSizeTransition hexutil.Uint64 `json:"maxCodeSizeTransition"` + EIP98Transition hexutil.Uint64 `json:"eip98Transition"` + EIP150Transition hexutil.Uint64 `json:"eip150Transition"` + EIP160Transition hexutil.Uint64 `json:"eip160Transition"` + EIP161abcTransition hexutil.Uint64 `json:"eip161abcTransition"` + EIP161dTransition hexutil.Uint64 `json:"eip161dTransition"` + EIP155Transition hexutil.Uint64 `json:"eip155Transition"` + EIP140Transition hexutil.Uint64 `json:"eip140Transition"` + EIP211Transition hexutil.Uint64 `json:"eip211Transition"` + EIP214Transition hexutil.Uint64 `json:"eip214Transition"` + EIP658Transition hexutil.Uint64 `json:"eip658Transition"` + EIP145Transition hexutil.Uint64 `json:"eip145Transition"` + EIP1014Transition hexutil.Uint64 `json:"eip1014Transition"` + EIP1052Transition hexutil.Uint64 `json:"eip1052Transition"` + EIP1283Transition hexutil.Uint64 `json:"eip1283Transition"` + EIP1283DisableTransition hexutil.Uint64 `json:"eip1283DisableTransition"` } `json:"params"` Genesis struct { @@ -347,6 +348,11 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin if num := genesis.Config.ConstantinopleBlock; num != nil { spec.setConstantinople(num) } + // ConstantinopleFix (remove eip-1283) + if num := genesis.Config.PetersburgBlock; num != nil { + spec.setConstantinopleFix(num) + } + spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) @@ -441,6 +447,10 @@ func (spec *parityChainSpec) setConstantinople(num *big.Int) { spec.Params.EIP1283Transition = n } +func (spec *parityChainSpec) setConstantinopleFix(num *big.Int) { + spec.Params.EIP1283DisableTransition = hexutil.Uint64(num.Uint64()) +} + // pyEthereumGenesisSpec represents the genesis specification format used by the // Python Ethereum implementation. type pyEthereumGenesisSpec struct { diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index cb3ed6e71c..9a77587b4a 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -608,30 +608,31 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da bootPython[i] = "'" + boot + "'" } template.Must(template.New("").Parse(dashboardContent)).Execute(indexfile, map[string]interface{}{ - "Network": network, - "NetworkID": conf.Genesis.Config.ChainID, - "NetworkTitle": strings.Title(network), - "EthstatsPage": config.ethstats, - "ExplorerPage": config.explorer, - "WalletPage": config.wallet, - "FaucetPage": config.faucet, - "GethGenesis": network + ".json", - "Bootnodes": conf.bootnodes, - "BootnodesFlat": strings.Join(conf.bootnodes, ","), - "Ethstats": statsLogin, - "Ethash": conf.Genesis.Config.Ethash != nil, - "CppGenesis": network + "-cpp.json", - "CppBootnodes": strings.Join(bootCpp, " "), - "HarmonyGenesis": network + "-harmony.json", - "HarmonyBootnodes": strings.Join(bootHarmony, " "), - "ParityGenesis": network + "-parity.json", - "PythonGenesis": network + "-python.json", - "PythonBootnodes": strings.Join(bootPython, ","), - "Homestead": conf.Genesis.Config.HomesteadBlock, - "Tangerine": conf.Genesis.Config.EIP150Block, - "Spurious": conf.Genesis.Config.EIP155Block, - "Byzantium": conf.Genesis.Config.ByzantiumBlock, - "Constantinople": conf.Genesis.Config.ConstantinopleBlock, + "Network": network, + "NetworkID": conf.Genesis.Config.ChainID, + "NetworkTitle": strings.Title(network), + "EthstatsPage": config.ethstats, + "ExplorerPage": config.explorer, + "WalletPage": config.wallet, + "FaucetPage": config.faucet, + "GethGenesis": network + ".json", + "Bootnodes": conf.bootnodes, + "BootnodesFlat": strings.Join(conf.bootnodes, ","), + "Ethstats": statsLogin, + "Ethash": conf.Genesis.Config.Ethash != nil, + "CppGenesis": network + "-cpp.json", + "CppBootnodes": strings.Join(bootCpp, " "), + "HarmonyGenesis": network + "-harmony.json", + "HarmonyBootnodes": strings.Join(bootHarmony, " "), + "ParityGenesis": network + "-parity.json", + "PythonGenesis": network + "-python.json", + "PythonBootnodes": strings.Join(bootPython, ","), + "Homestead": conf.Genesis.Config.HomesteadBlock, + "Tangerine": conf.Genesis.Config.EIP150Block, + "Spurious": conf.Genesis.Config.EIP155Block, + "Byzantium": conf.Genesis.Config.ByzantiumBlock, + "Constantinople": conf.Genesis.Config.ConstantinopleBlock, + "ConstantinopleFix": conf.Genesis.Config.PetersburgBlock, }) files[filepath.Join(workdir, "index.html")] = indexfile.Bytes() diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 95da5bd4f3..8abfe7c419 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -223,6 +223,10 @@ func (w *wizard) manageGenesis() { fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock) w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) + fmt.Println() + fmt.Printf("Which block should Constantinople-Fix (remove EIP-1283) come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock) + w.conf.Genesis.Config.PetersburgBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) + out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") fmt.Printf("Chain configuration updated:\n\n%s\n", out) diff --git a/cmd/swarm/bootnodes.go b/cmd/swarm/bootnodes.go index cbba9970da..ce3cd5288e 100644 --- a/cmd/swarm/bootnodes.go +++ b/cmd/swarm/bootnodes.go @@ -17,61 +17,8 @@ package main var SwarmBootnodes = []string{ - // Foundation Swarm Gateway Cluster - "enode://e5c6f9215c919a5450a7b8c14c22535607b69f2c8e1e7f6f430cb25d7a2c27cd1df4c4f18ad7c1d7e5162e271ffcd3f20b1a1467fb6e790e7d727f3b2193de97@52.232.7.187:30399", - "enode://9b2fe07e69ccc7db5fef15793dab7d7d2e697ed92132d6e9548218e68a34613a8671ad03a6658d862b468ed693cae8a0f8f8d37274e4a657ffb59ca84676e45b@52.232.7.187:30400", - "enode://76c1059162c93ef9df0f01097c824d17c492634df211ef4c806935b349082233b63b90c23970254b3b7138d630400f7cf9b71e80355a446a8b733296cb04169a@52.232.7.187:30401", - "enode://ce46bbe2a8263145d65252d52da06e000ad350ed09c876a71ea9544efa42f63c1e1b6cc56307373aaad8f9dd069c90d0ed2dd1530106200e16f4ca681dd8ae2d@52.232.7.187:30402", - "enode://f431e0d6008a6c35c6e670373d828390c8323e53da8158e7bfc43cf07e632cc9e472188be8df01decadea2d4a068f1428caba769b632554a8fb0607bc296988f@52.232.7.187:30403", - "enode://174720abfff83d7392f121108ae50ea54e04889afe020df883655c0f6cb95414db945a0228d8982fe000d86fc9f4b7669161adc89cd7cd56f78f01489ab2b99b@52.232.7.187:30404", - "enode://2ae89be4be61a689b6f9ecee4360a59e185e010ab750f14b63b4ae43d4180e872e18e3437d4386ce44875dc7cc6eb761acba06412fe3178f3dac1dab3b65703e@52.232.7.187:30405", - "enode://24abebe1c0e6d75d6052ce3219a87be8573fd6397b4cb51f0773b83abba9b3d872bfb273cdc07389715b87adfac02f5235f5241442c5089802cbd8d42e310fce@52.232.7.187:30406", - "enode://d08dfa46bfbbdbcaafbb6e34abee4786610f6c91e0b76d7881f0334ac10dda41d8c1f2b6eedffb4493293c335c0ad46776443b2208d1fbbb9e1a90b25ee4eef2@52.232.7.187:30407", - "enode://8d95eb0f837d27581a43668ed3b8783d69dc4e84aa3edd7a0897e026155c8f59c8702fdc0375ee7bac15757c9c78e1315d9b73e4ce59c936db52ea4ae2f501c7@52.232.7.187:30408", - "enode://a5967cc804aebd422baaaba9f06f27c9e695ccab335b61088130f8cbe64e3cdf78793868c7051dfc06eecfe844fad54bc7f6dfaed9db3c7ecef279cb829c25fb@52.232.7.187:30409", - "enode://5f00134d81a8f2ebcc46f8766f627f492893eda48138f811b7de2168308171968f01710bca6da05764e74f14bae41652f554e6321f1aed85fa3461e89d075dbf@52.232.7.187:30410", - "enode://b2142b79b01a5aa66a5e23cc35e78219a8e97bc2412a6698cee24ae02e87078b725d71730711bd62e25ff1aa8658c6633778af8ac14c63814a337c3dd0ebda9f@52.232.7.187:30411", - "enode://1ffa7651094867d6486ce3ef46d27a052c2cb968b618346c6df7040322c7efc3337547ba85d4cbba32e8b31c42c867202554735c06d4c664b9afada2ed0c4b3c@52.232.7.187:30412", - "enode://129e0c3d5f5df12273754f6f703d2424409fa4baa599e0b758c55600169313887855e75b082028d2302ec034b303898cd697cc7ae8256ba924ce927510da2c8d@52.232.7.187:30413", - "enode://419e2dc0d2f5b022cf16b0e28842658284909fa027a0fbbb5e2b755e7f846ea02a8f0b66a7534981edf6a7bcf8a14855344c6668e2cd4476ccd35a11537c9144@52.232.7.187:30414", - "enode://23d55ad900583231b91f2f62e3f72eb498b342afd58b682be3af052eed62b5651094471065981de33d8786f075f05e3cca499503b0ac8ae84b2a06e99f5b0723@52.232.7.187:30415", - "enode://bc56e4158c00e9f616d7ea533def20a89bef959df4e62a768ff238ff4e1e9223f57ecff969941c20921bad98749baae311c0fbebce53bf7bbb9d3dc903640990@52.232.7.187:30416", - "enode://433ce15199c409875e7e72fffd69fdafe746f17b20f0d5555281722a65fde6c80328fab600d37d8624509adc072c445ce0dad4a1c01cff6acf3132c11d429d4d@52.232.7.187:30417", - "enode://632ee95b8f0eac51ef89ceb29313fef3a60050181d66a6b125583b1a225a7694b252edc016efb58aa3b251da756cb73280842a022c658ed405223b2f58626343@52.232.7.187:30418", - "enode://4a0f9bcff7a4b9ee453fb298d0fb222592efe121512e30cd72fef631beb8c6a15153a1456eb073ee18551c0e003c569651a101892dc4124e90b933733a498bb5@52.232.7.187:30419", - "enode://f0d80fbc72d16df30e19aac3051eb56a7aff0c8367686702e01ea132d8b0b3ee00cadd6a859d2cca98ec68d3d574f8a8a87dba2347ec1e2818dc84bc3fa34fae@52.232.7.187:30420", - "enode://a199146906e4f9f2b94b195a8308d9a59a3564b92efaab898a4243fe4c2ad918b7a8e4853d9d901d94fad878270a2669d644591299c3d43de1b298c00b92b4a7@52.232.7.187:30421", - "enode://052036ea8736b37adbfb684d90ce43e11b3591b51f31489d7c726b03618dea4f73b1e659deb928e6bf40564edcdcf08351643f42db3d4ca1c2b5db95dad59e94@52.232.7.187:30422", - "enode://460e2b8c6da8f12fac96c836e7d108f4b7ec55a1c64631bb8992339e117e1c28328fee83af863196e20af1487a655d13e5ceba90e980e92502d5bac5834c1f71@52.232.7.187:30423", - "enode://6d2cdd13741b2e72e9031e1b93c6d9a4e68de2844aa4e939f6a8a8498a7c1d7e2ee4c64217e92a6df08c9a32c6764d173552810ef1bd2ecb356532d389dd2136@52.232.7.187:30424", - "enode://62105fc25ce2cd5b299647f47eaa9211502dc76f0e9f461df915782df7242ac3223e3db04356ae6ed2977ccac20f0b16864406e9ca514a40a004cb6a5d0402aa@52.232.7.187:30425", - "enode://e0e388fc520fd493c33f0ce16685e6f98fb6aec28f2edc14ee6b179594ee519a896425b0025bb6f0e182dd3e468443f19c70885fbc66560d000093a668a86aa8@52.232.7.187:30426", - "enode://63f3353a72521ea10022127a4fe6b4acbef197c3fe668fd9f4805542d8a6fcf79f6335fbab62d180a35e19b739483e740858b113fdd7c13a26ad7b4e318a5aef@52.232.7.187:30427", - "enode://33a42b927085678d4aefd4e70b861cfca6ef5f6c143696c4f755973fd29e64c9e658cad57a66a687a7a156da1e3688b1fbdd17bececff2ee009fff038fa5666b@52.232.7.187:30428", - "enode://259ab5ab5c1daee3eab7e3819ab3177b82d25c29e6c2444fdd3f956e356afae79a72840ccf2d0665fe82c81ebc3b3734da1178ac9fd5d62c67e674b69f86b6be@52.232.7.187:30429", - "enode://558bccad7445ce3fd8db116ed6ab4aed1324fdbdac2348417340c1764dc46d46bffe0728e5b7d5c36f12e794c289f18f57f08f085d2c65c9910a5c7a65b6a66a@52.232.7.187:30430", - "enode://abe60937a0657ffded718e3f84a32987286983be257bdd6004775c4b525747c2b598f4fac49c8de324de5ce75b22673fa541a7ce2d555fb7f8ca325744ae3577@52.232.7.187:30431", - "enode://bce6f0aaa5b230742680084df71d4f026b3eff7f564265599216a1b06b765303fdc9325de30ffd5dfdaf302ce4b14322891d2faea50ce2ca298d7409f5858339@52.232.7.187:30432", - "enode://21b957c4e03277d42be6660730ec1b93f540764f26c6abdb54d006611139c7081248486206dfbf64fcaffd62589e9c6b8ea77a5297e4b21a605f1bcf49483ed0@52.232.7.187:30433", - "enode://ff104e30e64f24c3d7328acee8b13354e5551bc8d60bb25ecbd9632d955c7e34bb2d969482d173355baad91c8282f8b592624eb3929151090da3b4448d4d58fb@52.232.7.187:30434", - "enode://c76e2b5f81a521bceaec1518926a21380a345df9cf463461562c6845795512497fb67679e155fc96a74350f8b78de8f4c135dd52b106dbbb9795452021d09ea5@52.232.7.187:30435", - "enode://3288fd860105164f3e9b69934c4eb18f7146cfab31b5a671f994e21a36e9287766e5f9f075aefbc404538c77f7c2eb2a4495020a7633a1c3970d94e9fa770aeb@52.232.7.187:30436", - "enode://6cea859c7396d46b20cfcaa80f9a11cd112f8684f2f782f7b4c0e1e0af9212113429522075101923b9b957603e6c32095a6a07b5e5e35183c521952ee108dfaf@52.232.7.187:30437", - "enode://f628ec56e4ca8317cc24cc4ac9b27b95edcce7b96e1c7f3b53e30de4a8580fe44f2f0694a513bdb0a431acaf2824074d6ace4690247bbc34c14f426af8c056ea@52.232.7.187:30438", - "enode://055ec8b26fc105c4f97970a1cce9773a5e34c03f511b839db742198a1c571e292c54aa799e9afb991cc8a560529b8cdf3e0c344bc6c282aff2f68eec59361ddf@52.232.7.187:30439", - "enode://48cb0d430c328974226aa33a931d8446cd5a8d40f3ead8f4ce7ad60faa1278192eb6d58bed91258d63e81f255fc107eec2425ce2ae8b22350dd556076e160610@52.232.7.187:30440", - "enode://3fadb7af7f770d5ffc6b073b8d42834bebb18ce1fe8a4fe270d2b799e7051327093960dc61d9a18870db288f7746a0e6ea2a013cd6ab0e5f97ca08199473aace@52.232.7.187:30441", - "enode://a5d7168024c9992769cf380ffa559a64b4f39a29d468f579559863814eb0ae0ed689ac0871a3a2b4c78b03297485ec322d578281131ef5d5c09a4beb6200a97a@52.232.7.187:30442", - "enode://9c57744c5b2c2d71abcbe80512652f9234d4ab041b768a2a886ab390fe6f184860f40e113290698652d7e20a8ac74d27ac8671db23eb475b6c5e6253e4693bf8@52.232.7.187:30443", - "enode://daca9ff0c3176045a0e0ed228dee00ec86bc0939b135dc6b1caa23745d20fd0332e1ee74ad04020e89df56c7146d831a91b89d15ca3df05ba7618769fefab376@52.232.7.187:30444", - "enode://a3f6af59428cb4b9acb198db15ef5554fa43c2b0c18e468a269722d64a27218963a2975eaf82750b6262e42192b5e3669ea51337b4cda62b33987981bc5e0c1a@52.232.7.187:30445", - "enode://fe571422fa4651c3354c85dac61911a6a6520dd3c0332967a49d4133ca30e16a8a4946fa73ca2cb5de77917ea701a905e1c3015b2f4defcd53132b61cc84127a@52.232.7.187:30446", - - // Mainframe - "enode://ee9a5a571ea6c8a59f9a8bb2c569c865e922b41c91d09b942e8c1d4dd2e1725bd2c26149da14de1f6321a2c6fdf1e07c503c3e093fb61696daebf74d6acd916b@54.186.219.160:30399", - "enode://a03f0562ecb8a992ad5242345535e73483cdc18ab934d36bf24b567d43447c2cea68f89f1d51d504dd13acc30f24ebce5a150bea2ccb1b722122ce4271dc199d@52.67.248.147:30399", - "enode://e2cbf9eafd85903d3b1c56743035284320695e0072bc8d7396e0542aa5e1c321b236f67eab66b79c2f15d4447fa4bbe74dd67d0467da23e7eb829f60ec8a812b@13.58.169.1:30399", - "enode://8b8c6bda6047f1cad9fab2db4d3d02b7aa26279902c32879f7bcd4a7d189fee77fdc36ee151ce6b84279b4792e72578fd529d2274d014132465758fbfee51cee@13.209.13.15:30399", - "enode://63f6a8818927e429585287cf2ca0cb9b11fa990b7b9b331c2962cdc6f21807a2473b26e8256225c26caff70d7218e59586d704d49061452c6852e382c885d03c@35.154.106.174:30399", - "enode://ed4bd3b794ed73f18e6dcc70c6624dfec63b5654f6ab54e8f40b16eff8afbd342d4230e099ddea40e84423f81b2d2ea79799dc345257b1fec6f6c422c9d008f7@52.213.20.99:30399", + // EF Swarm Bootnode - AWS - eu-central-1 + "enode://4c113504601930bf2000c29bcd98d1716b6167749f58bad703bae338332fe93cc9d9204f08afb44100dc7bea479205f5d162df579f9a8f76f8b402d339709023@3.122.203.99:30301", + // EF Swarm Bootnode - AWS - us-west-2 + "enode://89f2ede3371bff1ad9f2088f2012984e280287a4e2b68007c2a6ad994909c51886b4a8e9e2ecc97f9910aca538398e0a5804b0ee80a187fde1ba4f32626322ba@52.35.212.179:30301", } diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 3eea3057b3..0203a67984 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -79,6 +79,7 @@ const ( SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" + SWARM_ENV_BOOTNODE_MODE = "SWARM_BOOTNODE_MODE" SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" GETH_ENV_DATADIR = "GETH_DATADIR" @@ -164,10 +165,9 @@ func configFileOverride(config *bzzapi.Config, ctx *cli.Context) (*bzzapi.Config return config, err } -//override the current config with whatever is provided through the command line -//most values are not allowed a zero value (empty string), if not otherwise noted +// cmdLineOverride overrides the current config with whatever is provided through the command line +// most values are not allowed a zero value (empty string), if not otherwise noted func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Config { - if keyid := ctx.GlobalString(SwarmAccountFlag.Name); keyid != "" { currentConfig.BzzAccount = keyid } @@ -258,14 +258,17 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.LocalStoreParams.CacheCapacity = storeCacheCapacity } + if ctx.GlobalIsSet(SwarmBootnodeModeFlag.Name) { + currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name) + } + return currentConfig } -//override the current config with whatver is provided in environment variables -//most values are not allowed a zero value (empty string), if not otherwise noted +// envVarsOverride overrides the current config with whatver is provided in environment variables +// most values are not allowed a zero value (empty string), if not otherwise noted func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { - if keyid := os.Getenv(SWARM_ENV_ACCOUNT); keyid != "" { currentConfig.BzzAccount = keyid } @@ -364,6 +367,14 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { currentConfig.Cors = cors } + if bm := os.Getenv(SWARM_ENV_BOOTNODE_MODE); bm != "" { + bootnodeMode, err := strconv.ParseBool(bm) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_BOOTNODE_MODE, err) + } + currentConfig.BootnodeMode = bootnodeMode + } + return currentConfig } diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go index 12edc8cc98..4c186cc310 100644 --- a/cmd/swarm/flags.go +++ b/cmd/swarm/flags.go @@ -156,6 +156,10 @@ var ( Name: "compressed", Usage: "Prints encryption keys in compressed form", } + SwarmBootnodeModeFlag = cli.BoolFlag{ + Name: "bootnode-mode", + Usage: "Run Swarm in Bootnode mode", + } SwarmFeedNameFlag = cli.StringFlag{ Name: "name", Usage: "User-defined name for the new feed, limited to 32 characters. If combined with topic, it will refer to a subtopic with this name", diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index ccbb24eec3..53888b615f 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -66,9 +66,10 @@ OPTIONS: {{end}}{{end}} ` -var ( - gitCommit string // Git SHA1 commit hash of the release (set via linker flags) -) +// Git SHA1 commit hash of the release (set via linker flags) +// this variable will be assigned if corresponding parameter is passed with install, but not with test +// e.g.: go install -ldflags "-X main.gitCommit=ed1312d01b19e04ef578946226e5d8069d5dfd5a" ./cmd/swarm +var gitCommit string //declare a few constant error messages, useful for later error check comparisons in test var ( @@ -89,6 +90,7 @@ var defaultNodeConfig = node.DefaultConfig // This init function sets defaults so cmd/swarm can run alongside geth. func init() { + sv.GitCommit = gitCommit defaultNodeConfig.Name = clientIdentifier defaultNodeConfig.Version = sv.VersionWithCommit(gitCommit) defaultNodeConfig.P2P.ListenAddr = ":30399" @@ -154,7 +156,6 @@ func init() { utils.BootnodesFlag, utils.KeyStoreDirFlag, utils.ListenPortFlag, - utils.NoDiscoverFlag, utils.DiscoveryV5Flag, utils.NetrestrictFlag, utils.NodeKeyFileFlag, @@ -187,6 +188,8 @@ func init() { SwarmUploadDefaultPath, SwarmUpFromStdinFlag, SwarmUploadMimeType, + // bootnode mode + SwarmBootnodeModeFlag, // storage flags SwarmStorePath, SwarmStoreCapacity, @@ -227,12 +230,17 @@ func main() { func keys(ctx *cli.Context) error { privateKey := getPrivKey(ctx) - pub := hex.EncodeToString(crypto.FromECDSAPub(&privateKey.PublicKey)) + pubkey := crypto.FromECDSAPub(&privateKey.PublicKey) + pubkeyhex := hex.EncodeToString(pubkey) pubCompressed := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)) + bzzkey := crypto.Keccak256Hash(pubkey).Hex() + if !ctx.Bool(SwarmCompressedFlag.Name) { - fmt.Println(fmt.Sprintf("publicKey=%s", pub)) + fmt.Println(fmt.Sprintf("bzzkey=%s", bzzkey[2:])) + fmt.Println(fmt.Sprintf("publicKey=%s", pubkeyhex)) } fmt.Println(fmt.Sprintf("publicKeyCompressed=%s", pubCompressed)) + return nil } @@ -272,6 +280,10 @@ func bzzd(ctx *cli.Context) error { setSwarmBootstrapNodes(ctx, &cfg) //setup the ethereum node utils.SetNodeConfig(ctx, &cfg) + + //always disable discovery from p2p package - swarm discovery is done with the `hive` protocol + cfg.P2P.NoDiscovery = true + stack, err := node.New(&cfg) if err != nil { utils.Fatalf("can't create node: %v", err) @@ -294,6 +306,15 @@ func bzzd(ctx *cli.Context) error { stack.Stop() }() + // add swarm bootnodes, because swarm doesn't use p2p package's discovery discv5 + go func() { + s := stack.Server() + + for _, n := range cfg.P2P.BootstrapNodes { + s.AddPeer(n) + } + }() + stack.Wait() return nil } diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 680d238d06..4a6a56d9b8 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -254,7 +254,6 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode { node.Cmd = runSwarm(t, "--port", p2pPort, "--nat", "extip:127.0.0.1", - "--nodiscover", "--datadir", dir, "--ipcpath", conf.IPCPath, "--ens-api", "", @@ -330,7 +329,6 @@ func newTestNode(t *testing.T, dir string) *testNode { node.Cmd = runSwarm(t, "--port", p2pPort, "--nat", "extip:127.0.0.1", - "--nodiscover", "--datadir", dir, "--ipcpath", conf.IPCPath, "--ens-api", "", diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 66cecdc5c7..ebd0c97155 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -130,7 +130,7 @@ func main() { swarmmetrics.MetricsInfluxDBDatabaseFlag, swarmmetrics.MetricsInfluxDBUsernameFlag, swarmmetrics.MetricsInfluxDBPasswordFlag, - swarmmetrics.MetricsInfluxDBHostTagFlag, + swarmmetrics.MetricsInfluxDBTagsFlag, }...) app.Flags = append(app.Flags, tracing.Flags...) @@ -148,6 +148,12 @@ func main() { Usage: "feed update generate, upload and sync", Action: cliFeedUploadAndSync, }, + { + Name: "upload_speed", + Aliases: []string{"u"}, + Usage: "measure upload speed", + Action: cliUploadSpeed, + }, } sort.Sort(cli.FlagsByName(app.Flags)) @@ -177,13 +183,14 @@ func emitMetrics(ctx *cli.Context) error { database = ctx.GlobalString(swarmmetrics.MetricsInfluxDBDatabaseFlag.Name) username = ctx.GlobalString(swarmmetrics.MetricsInfluxDBUsernameFlag.Name) password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name) + tags = ctx.GlobalString(swarmmetrics.MetricsInfluxDBTagsFlag.Name) ) - return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", map[string]string{ - "host": hosttag, - "version": gitCommit, - "filesize": fmt.Sprintf("%v", filesize), - }) + + tagsMap := utils.SplitTagsFlag(tags) + tagsMap["version"] = gitCommit + tagsMap["filesize"] = fmt.Sprintf("%v", filesize) + + return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", tagsMap) } return nil diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index d605f79a3a..0fc86c55d7 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -49,6 +49,10 @@ func generateEndpoints(scheme string, cluster string, app string, from int, to i for port := from; port < to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) } + } else if cluster == "private-internal" { + for port := from; port < to; port++ { + endpoints = append(endpoints, fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, port)) + } } else { for port := from; port < to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) @@ -86,9 +90,8 @@ func cliUploadAndSync(c *cli.Context) error { func uploadAndSync(c *cli.Context) error { defer func(now time.Time) { totalTime := time.Since(now) - log.Info("total time", "time", totalTime, "kb", filesize) - metrics.GetOrRegisterCounter("upload-and-sync.total-time", nil).Inc(int64(totalTime)) + metrics.GetOrRegisterResettingTimer("upload-and-sync.total-time", nil).Update(totalTime) }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) @@ -103,7 +106,7 @@ func uploadAndSync(c *cli.Context) error { log.Error(err.Error()) return err } - metrics.GetOrRegisterCounter("upload-and-sync.upload-time", nil).Inc(int64(time.Since(t1))) + metrics.GetOrRegisterResettingTimer("upload-and-sync.upload-time", nil).UpdateSince(t1) fhash, err := digest(bytes.NewReader(randomBytes)) if err != nil { @@ -125,30 +128,28 @@ func uploadAndSync(c *cli.Context) error { for { start := time.Now() err := fetch(hash, endpoint, fhash, ruid) - fetchTime := time.Since(start) if err != nil { continue } - metrics.GetOrRegisterMeter("upload-and-sync.single.fetch-time", nil).Mark(int64(fetchTime)) + metrics.GetOrRegisterResettingTimer("upload-and-sync.single.fetch-time", nil).UpdateSince(start) wg.Done() return } }(endpoints[randIndex], ruid) } else { - for _, endpoint := range endpoints { + for _, endpoint := range endpoints[1:] { ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { for { start := time.Now() err := fetch(hash, endpoint, fhash, ruid) - fetchTime := time.Since(start) if err != nil { continue } - metrics.GetOrRegisterMeter("upload-and-sync.each.fetch-time", nil).Mark(int64(fetchTime)) + metrics.GetOrRegisterResettingTimer("upload-and-sync.each.fetch-time", nil).UpdateSince(start) wg.Done() return } @@ -166,8 +167,6 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error { ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch") defer sp.Finish() - log.Trace("sleeping", "ruid", ruid) - time.Sleep(3 * time.Second) log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) var tn time.Time diff --git a/cmd/swarm/swarm-smoke/upload_speed.go b/cmd/swarm/swarm-smoke/upload_speed.go new file mode 100644 index 0000000000..4a355baf82 --- /dev/null +++ b/cmd/swarm/swarm-smoke/upload_speed.go @@ -0,0 +1,98 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "bytes" + "fmt" + "os" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/testutil" + + cli "gopkg.in/urfave/cli.v1" +) + +var endpoint string + +//just use the first endpoint +func generateEndpoint(scheme string, cluster string, app string, from int) { + if cluster == "prod" { + endpoint = fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, from) + } else if cluster == "private-internal" { + endpoint = fmt.Sprintf("%s://swarm-private-internal-%v:8500", scheme, from) + } else { + endpoint = fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, from, cluster) + } +} + +func cliUploadSpeed(c *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + + metrics.GetOrRegisterCounter("upload-speed", nil).Inc(1) + + errc := make(chan error) + go func() { + errc <- uploadSpeed(c) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter("upload-speed.fail", nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter("upload-speed.timeout", nil).Inc(1) + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +func uploadSpeed(c *cli.Context) error { + defer func(now time.Time) { + totalTime := time.Since(now) + + log.Info("total time", "time", totalTime, "kb", filesize) + metrics.GetOrRegisterCounter("upload-speed.total-time", nil).Inc(int64(totalTime)) + }(time.Now()) + + generateEndpoint(scheme, cluster, appName, from) + seed := int(time.Now().UnixNano() / 1e6) + log.Info("uploading to "+endpoint, "seed", seed) + + randomBytes := testutil.RandomBytes(seed, filesize*1000) + + t1 := time.Now() + hash, err := upload(&randomBytes, endpoint) + if err != nil { + log.Error(err.Error()) + return err + } + metrics.GetOrRegisterCounter("upload-speed.upload-time", nil).Inc(int64(time.Since(t1))) + + fhash, err := digest(bytes.NewReader(randomBytes)) + if err != nil { + log.Error(err.Error()) + return err + } + + log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) + return nil +} diff --git a/cmd/swarm/swarm-snapshot/create.go b/cmd/swarm/swarm-snapshot/create.go new file mode 100644 index 0000000000..127fde8ae2 --- /dev/null +++ b/cmd/swarm/swarm-snapshot/create.go @@ -0,0 +1,157 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/network/simulation" + cli "gopkg.in/urfave/cli.v1" +) + +// create is used as the entry function for "create" app command. +func create(ctx *cli.Context) error { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + + if len(ctx.Args()) < 1 { + return errors.New("argument should be the filename to verify or write-to") + } + filename, err := touchPath(ctx.Args()[0]) + if err != nil { + return err + } + return createSnapshot(filename, ctx.Int("nodes"), strings.Split(ctx.String("services"), ",")) +} + +// createSnapshot creates a new snapshot on filesystem with provided filename, +// number of nodes and service names. +func createSnapshot(filename string, nodes int, services []string) (err error) { + log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services) + + sim := simulation.New(map[string]simulation.ServiceFunc{ + "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { + addr := network.NewAddr(ctx.Config.Node()) + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + hp := network.NewHiveParams() + hp.KeepAliveInterval = time.Duration(200) * time.Millisecond + hp.Discovery = true // discovery must be enabled when creating a snapshot + + config := &network.BzzConfig{ + OverlayAddr: addr.Over(), + UnderlayAddr: addr.Under(), + HiveParams: hp, + } + return network.NewBzz(config, kad, nil, nil, nil), nil, nil + }, + }) + defer sim.Close() + + _, err = sim.AddNodes(nodes) + if err != nil { + return fmt.Errorf("add nodes: %v", err) + } + + err = sim.Net.ConnectNodesRing(nil) + if err != nil { + return fmt.Errorf("connect nodes: %v", err) + } + + ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancelSimRun() + if _, err := sim.WaitTillHealthy(ctx); err != nil { + return fmt.Errorf("wait for healthy kademlia: %v", err) + } + + var snap *simulations.Snapshot + if len(services) > 0 { + // If service names are provided, include them in the snapshot. + // But, check if "bzz" service is not among them to remove it + // form the snapshot as it exists on snapshot creation. + var removeServices []string + var wantBzz bool + for _, s := range services { + if s == "bzz" { + wantBzz = true + break + } + } + if !wantBzz { + removeServices = []string{"bzz"} + } + snap, err = sim.Net.SnapshotWithServices(services, removeServices) + } else { + snap, err = sim.Net.Snapshot() + } + if err != nil { + return fmt.Errorf("create snapshot: %v", err) + } + jsonsnapshot, err := json.Marshal(snap) + if err != nil { + return fmt.Errorf("json encode snapshot: %v", err) + } + return ioutil.WriteFile(filename, jsonsnapshot, 0666) +} + +// touchPath creates an empty file and all subdirectories +// that are missing. +func touchPath(filename string) (string, error) { + if path.IsAbs(filename) { + if _, err := os.Stat(filename); err == nil { + // path exists, overwrite + return filename, nil + } + } + + d, f := path.Split(filename) + dir, err := filepath.Abs(filepath.Dir(os.Args[0])) + if err != nil { + return "", err + } + + _, err = os.Stat(path.Join(dir, filename)) + if err == nil { + // path exists, overwrite + return filename, nil + } + + dirPath := path.Join(dir, d) + filePath := path.Join(dirPath, f) + if d != "" { + err = os.MkdirAll(dirPath, os.ModeDir) + if err != nil { + return "", err + } + } + + return filePath, nil +} diff --git a/cmd/swarm/swarm-snapshot/create_test.go b/cmd/swarm/swarm-snapshot/create_test.go new file mode 100644 index 0000000000..c9445168d8 --- /dev/null +++ b/cmd/swarm/swarm-snapshot/create_test.go @@ -0,0 +1,143 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "runtime" + "sort" + "strconv" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/p2p/simulations" +) + +// TestSnapshotCreate is a high level e2e test that tests for snapshot generation. +// It runs a few "create" commands with different flag values and loads generated +// snapshot files to validate their content. +func TestSnapshotCreate(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + for _, v := range []struct { + name string + nodes int + services string + }{ + { + name: "defaults", + }, + { + name: "more nodes", + nodes: defaultNodes + 5, + }, + { + name: "services", + services: "stream,pss,zorglub", + }, + { + name: "services with bzz", + services: "bzz,pss", + }, + } { + t.Run(v.name, func(t *testing.T) { + t.Parallel() + + file, err := ioutil.TempFile("", "swarm-snapshot") + if err != nil { + t.Fatal(err) + } + defer os.Remove(file.Name()) + + if err = file.Close(); err != nil { + t.Error(err) + } + + args := []string{"create"} + if v.nodes > 0 { + args = append(args, "--nodes", strconv.Itoa(v.nodes)) + } + if v.services != "" { + args = append(args, "--services", v.services) + } + testCmd := runSnapshot(t, append(args, file.Name())...) + + testCmd.ExpectExit() + if code := testCmd.ExitStatus(); code != 0 { + t.Fatalf("command exit code %v, expected 0", code) + } + + f, err := os.Open(file.Name()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := f.Close() + if err != nil { + t.Error("closing snapshot file", "err", err) + } + }() + + b, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + var snap simulations.Snapshot + err = json.Unmarshal(b, &snap) + if err != nil { + t.Fatal(err) + } + + wantNodes := v.nodes + if wantNodes == 0 { + wantNodes = defaultNodes + } + gotNodes := len(snap.Nodes) + if gotNodes != wantNodes { + t.Errorf("got %v nodes, want %v", gotNodes, wantNodes) + } + + if len(snap.Conns) == 0 { + t.Error("no connections in a snapshot") + } + + var wantServices []string + if v.services != "" { + wantServices = strings.Split(v.services, ",") + } else { + wantServices = []string{"bzz"} + } + // sort service names so they can be comparable + // as strings to every node sorted services + sort.Strings(wantServices) + + for i, n := range snap.Nodes { + gotServices := n.Node.Config.Services + sort.Strings(gotServices) + if fmt.Sprint(gotServices) != fmt.Sprint(wantServices) { + t.Errorf("got services %v for node %v, want %v", gotServices, i, wantServices) + } + } + + }) + } +} diff --git a/cmd/swarm/swarm-snapshot/main.go b/cmd/swarm/swarm-snapshot/main.go new file mode 100644 index 0000000000..184727e4d7 --- /dev/null +++ b/cmd/swarm/swarm-snapshot/main.go @@ -0,0 +1,82 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "os" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + cli "gopkg.in/urfave/cli.v1" +) + +var gitCommit string // Git SHA1 commit hash of the release (set via linker flags) + +// default value for "create" command --nodes flag +const defaultNodes = 10 + +func main() { + err := newApp().Run(os.Args) + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } +} + +// newApp construct a new instance of Swarm Snapshot Utility. +// Method Run is called on it in the main function and in tests. +func newApp() (app *cli.App) { + app = utils.NewApp(gitCommit, "Swarm Snapshot Utility") + + app.Name = "swarm-snapshot" + app.Usage = "" + + // app flags (for all commands) + app.Flags = []cli.Flag{ + cli.IntFlag{ + Name: "verbosity", + Value: 1, + Usage: "verbosity level", + }, + } + + app.Commands = []cli.Command{ + { + Name: "create", + Aliases: []string{"c"}, + Usage: "create a swarm snapshot", + Action: create, + // Flags only for "create" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.IntFlag{ + Name: "nodes", + Value: defaultNodes, + Usage: "number of nodes", + }, + cli.StringFlag{ + Name: "services", + Value: "bzz", + Usage: "comma separated list of services to boot the nodes with", + }, + ), + }, + } + + return app +} diff --git a/cmd/swarm/swarm-snapshot/run_test.go b/cmd/swarm/swarm-snapshot/run_test.go new file mode 100644 index 0000000000..d9a041597e --- /dev/null +++ b/cmd/swarm/swarm-snapshot/run_test.go @@ -0,0 +1,49 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +func init() { + reexec.Register("swarm-snapshot", func() { + if err := newApp().Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) +} + +func runSnapshot(t *testing.T, args ...string) *cmdtest.TestCmd { + tt := cmdtest.NewTestCmd(t, nil) + tt.Run("swarm-snapshot", args...) + return tt +} + +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 60e45d095a..3109bf962a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -19,6 +19,7 @@ package utils import ( "crypto/ecdsa" + "encoding/json" "fmt" "io/ioutil" "math/big" @@ -57,7 +58,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) var ( @@ -161,6 +162,23 @@ var ( Usage: "Document Root for HTTPClient file scheme", Value: DirectoryString{homeDir()}, } + ULCModeConfigFlag = cli.StringFlag{ + Name: "ulc.config", + Usage: "Config file to use for ultra light client mode", + } + OnlyAnnounceModeFlag = cli.BoolFlag{ + Name: "ulc.onlyannounce", + Usage: "ULC server sends announcements only", + } + ULCMinTrustedFractionFlag = cli.IntFlag{ + Name: "ulc.fraction", + Usage: "Minimum % of trusted ULC servers required to announce a new head", + } + ULCTrustedNodesFlag = cli.StringFlag{ + Name: "ulc.trusted", + Usage: "List of trusted ULC servers", + } + defaultSyncMode = eth.DefaultConfig.SyncMode SyncModeFlag = TextMarshalerFlag{ Name: "syncmode", @@ -435,6 +453,30 @@ var ( Usage: "HTTP-RPC server listening port", Value: node.DefaultHTTPPort, } + GraphQLEnabledFlag = cli.BoolFlag{ + Name: "graphql", + Usage: "Enable the GraphQL server", + } + GraphQLListenAddrFlag = cli.StringFlag{ + Name: "graphql.addr", + Usage: "GraphQL server listening interface", + Value: node.DefaultGraphQLHost, + } + GraphQLPortFlag = cli.IntFlag{ + Name: "graphql.port", + Usage: "GraphQL server listening port", + Value: node.DefaultGraphQLPort, + } + GraphQLCORSDomainFlag = cli.StringFlag{ + Name: "graphql.rpccorsdomain", + Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", + Value: "", + } + GraphQLVirtualHostsFlag = cli.StringFlag{ + Name: "graphql.rpcvhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: strings.Join(node.DefaultConfig.HTTPVirtualHosts, ","), + } RPCCORSDomainFlag = cli.StringFlag{ Name: "rpccorsdomain", Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", @@ -614,14 +656,14 @@ var ( Usage: "Password to authorize access to the database", Value: "test", } - // The `host` tag is part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. - // It is used so that we can group all nodes and average a measurement across all of them, but also so - // that we can select a specific node and inspect its measurements. + // Tags are part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. + // For example `host` tag could be used so that we can group all nodes and average a measurement + // across all of them, but also so that we can select a specific node and inspect its measurements. // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key - MetricsInfluxDBHostTagFlag = cli.StringFlag{ - Name: "metrics.influxdb.host.tag", - Usage: "InfluxDB `host` tag attached to all measurements", - Value: "localhost", + MetricsInfluxDBTagsFlag = cli.StringFlag{ + Name: "metrics.influxdb.tags", + Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements", + Value: "host=localhost", } EWASMInterpreterFlag = cli.StringFlag{ @@ -707,11 +749,13 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { cfg.BootstrapNodes = make([]*enode.Node, 0, len(urls)) for _, url := range urls { - node, err := enode.ParseV4(url) - if err != nil { - log.Crit("Bootstrap URL invalid", "enode", url, "err", err) + if url != "" { + node, err := enode.ParseV4(url) + if err != nil { + log.Crit("Bootstrap URL invalid", "enode", url, "err", err) + } + cfg.BootstrapNodes = append(cfg.BootstrapNodes, node) } - cfg.BootstrapNodes = append(cfg.BootstrapNodes, node) } } @@ -796,6 +840,24 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) { } } +// setGraphQL creates the GraphQL listener interface string from the set +// command line flags, returning empty if the GraphQL endpoint is disabled. +func setGraphQL(ctx *cli.Context, cfg *node.Config) { + if ctx.GlobalBool(GraphQLEnabledFlag.Name) && cfg.GraphQLHost == "" { + cfg.GraphQLHost = "127.0.0.1" + if ctx.GlobalIsSet(GraphQLListenAddrFlag.Name) { + cfg.GraphQLHost = ctx.GlobalString(GraphQLListenAddrFlag.Name) + } + } + cfg.GraphQLPort = ctx.GlobalInt(GraphQLPortFlag.Name) + if ctx.GlobalIsSet(GraphQLCORSDomainFlag.Name) { + cfg.GraphQLCors = splitAndTrim(ctx.GlobalString(GraphQLCORSDomainFlag.Name)) + } + if ctx.GlobalIsSet(GraphQLVirtualHostsFlag.Name) { + cfg.GraphQLVirtualHosts = splitAndTrim(ctx.GlobalString(GraphQLVirtualHostsFlag.Name)) + } +} + // setWS creates the WebSocket RPC listener interface string from the set // command line flags, returning empty if the HTTP endpoint is disabled. func setWS(ctx *cli.Context, cfg *node.Config) { @@ -829,6 +891,40 @@ func setIPC(ctx *cli.Context, cfg *node.Config) { } } +// SetULC setup ULC config from file if given. +func SetULC(ctx *cli.Context, cfg *eth.Config) { + // ULC config isn't loaded from global config and ULC config and ULC trusted nodes are not defined. + if cfg.ULC == nil && !(ctx.GlobalIsSet(ULCModeConfigFlag.Name) || ctx.GlobalIsSet(ULCTrustedNodesFlag.Name)) { + return + } + cfg.ULC = ð.ULCConfig{} + + path := ctx.GlobalString(ULCModeConfigFlag.Name) + if path != "" { + cfgData, err := ioutil.ReadFile(path) + if err != nil { + Fatalf("Failed to unmarshal ULC configuration: %v", err) + } + + err = json.Unmarshal(cfgData, &cfg.ULC) + if err != nil { + Fatalf("Failed to unmarshal ULC configuration: %s", err.Error()) + } + } + + if trustedNodes := ctx.GlobalString(ULCTrustedNodesFlag.Name); trustedNodes != "" { + cfg.ULC.TrustedServers = strings.Split(trustedNodes, ",") + } + + if trustedFraction := ctx.GlobalInt(ULCMinTrustedFractionFlag.Name); trustedFraction > 0 { + cfg.ULC.MinTrustedFraction = trustedFraction + } + if cfg.ULC.MinTrustedFraction <= 0 && cfg.ULC.MinTrustedFraction > 100 { + log.Error("MinTrustedFraction is invalid", "MinTrustedFraction", cfg.ULC.MinTrustedFraction, "Changed to default", eth.DefaultULCMinTrustedFraction) + cfg.ULC.MinTrustedFraction = eth.DefaultULCMinTrustedFraction + } +} + // makeDatabaseHandles raises out the number of allowed file handles per process // for Geth and returns half of the allowance to assign to the database. func makeDatabaseHandles() int { @@ -978,6 +1074,7 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { SetP2PConfig(ctx, &cfg.P2P) setIPC(ctx, cfg) setHTTP(ctx, cfg) + setGraphQL(ctx, cfg) setWS(ctx, cfg) setNodeUserIdent(ctx, cfg) @@ -1179,6 +1276,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(LightPeersFlag.Name) { cfg.LightPeers = ctx.GlobalInt(LightPeersFlag.Name) } + if ctx.GlobalIsSet(OnlyAnnounceModeFlag.Name) { + cfg.OnlyAnnounce = ctx.GlobalBool(OnlyAnnounceModeFlag.Name) + } if ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name) } @@ -1360,18 +1460,35 @@ func SetupMetrics(ctx *cli.Context) { database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(MetricsInfluxDBHostTagFlag.Name) ) if enableExport { + tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) + log.Info("Enabling metrics export to InfluxDB") - go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", map[string]string{ - "host": hosttag, - }) + + go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap) } } } +func SplitTagsFlag(tagsFlag string) map[string]string { + tags := strings.Split(tagsFlag, ",") + tagsMap := map[string]string{} + + for _, t := range tags { + if t != "" { + kv := strings.Split(t, "=") + + if len(kv) == 2 { + tagsMap[kv[0]] = kv[1] + } + } + } + + return tagsMap +} + // MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails. func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { var ( diff --git a/cmd/utils/flags_test.go b/cmd/utils/flags_test.go new file mode 100644 index 0000000000..adfdd0903e --- /dev/null +++ b/cmd/utils/flags_test.go @@ -0,0 +1,64 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// Package utils contains internal helper functions for go-ethereum commands. +package utils + +import ( + "reflect" + "testing" +) + +func Test_SplitTagsFlag(t *testing.T) { + tests := []struct { + name string + args string + want map[string]string + }{ + { + "2 tags case", + "host=localhost,bzzkey=123", + map[string]string{ + "host": "localhost", + "bzzkey": "123", + }, + }, + { + "1 tag case", + "host=localhost123", + map[string]string{ + "host": "localhost123", + }, + }, + { + "empty case", + "", + map[string]string{}, + }, + { + "garbage", + "smth=smthelse=123", + map[string]string{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := SplitTagsFlag(tt.args); !reflect.DeepEqual(got, tt.want) { + t.Errorf("splitTagsFlag() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/common/hexutil/json.go b/common/hexutil/json.go index fbc21241c8..777b08eca4 100644 --- a/common/hexutil/json.go +++ b/common/hexutil/json.go @@ -72,6 +72,25 @@ func (b Bytes) String() string { return Encode(b) } +// ImplementsGraphQLType returns true if Bytes implements the specified GraphQL type. +func (b Bytes) ImplementsGraphQLType(name string) bool { return name == "Bytes" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (b *Bytes) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + data, err := Decode(input) + if err != nil { + return err + } + *b = data + default: + err = fmt.Errorf("Unexpected type for Bytes: %v", input) + } + return err +} + // UnmarshalFixedJSON decodes the input as a string with 0x prefix. The length of out // determines the required input length. This function is commonly used to implement the // UnmarshalJSON method for fixed-size types. @@ -187,6 +206,25 @@ func (b *Big) String() string { return EncodeBig(b.ToInt()) } +// ImplementsGraphQLType returns true if Big implements the provided GraphQL type. +func (b Big) ImplementsGraphQLType(name string) bool { return name == "BigInt" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (b *Big) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + return b.UnmarshalText([]byte(input)) + case int32: + var num big.Int + num.SetInt64(int64(input)) + *b = Big(num) + default: + err = fmt.Errorf("Unexpected type for BigInt: %v", input) + } + return err +} + // Uint64 marshals/unmarshals as a JSON string with 0x prefix. // The zero value marshals as "0x0". type Uint64 uint64 @@ -234,6 +272,23 @@ func (b Uint64) String() string { return EncodeUint64(uint64(b)) } +// ImplementsGraphQLType returns true if Uint64 implements the provided GraphQL type. +func (b Uint64) ImplementsGraphQLType(name string) bool { return name == "Long" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (b *Uint64) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + return b.UnmarshalText([]byte(input)) + case int32: + *b = Uint64(input) + default: + err = fmt.Errorf("Unexpected type for Long: %v", input) + } + return err +} + // Uint marshals/unmarshals as a JSON string with 0x prefix. // The zero value marshals as "0x0". type Uint uint diff --git a/common/types.go b/common/types.go index 0f4892d287..48043788fd 100644 --- a/common/types.go +++ b/common/types.go @@ -141,6 +141,21 @@ func (h Hash) Value() (driver.Value, error) { return h[:], nil } +// ImplementsGraphQLType returns true if Hash implements the specified GraphQL type. +func (_ Hash) ImplementsGraphQLType(name string) bool { return name == "Bytes32" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (h *Hash) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + *h = HexToHash(input) + default: + err = fmt.Errorf("Unexpected type for Bytes32: %v", input) + } + return err +} + // UnprefixedHash allows marshaling a Hash without 0x prefix. type UnprefixedHash Hash @@ -268,6 +283,21 @@ func (a Address) Value() (driver.Value, error) { return a[:], nil } +// ImplementsGraphQLType returns true if Hash implements the specified GraphQL type. +func (a Address) ImplementsGraphQLType(name string) bool { return name == "Address" } + +// UnmarshalGraphQL unmarshals the provided GraphQL query data. +func (a *Address) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + *a = HexToAddress(input) + default: + err = fmt.Errorf("Unexpected type for Address: %v", input) + } + return err +} + // UnprefixedAddress allows marshaling an Address without 0x prefix. type UnprefixedAddress Address diff --git a/console/bridge.go b/console/bridge.go index b0b4d37985..33277cf6ee 100644 --- a/console/bridge.go +++ b/console/bridge.go @@ -105,9 +105,37 @@ func (b *bridge) OpenWallet(call otto.FunctionCall) (response otto.Value) { return val } // Wallet open failed, report error unless it's a PIN entry - if !strings.HasSuffix(err.Error(), usbwallet.ErrTrezorPINNeeded.Error()) { + if strings.HasSuffix(err.Error(), usbwallet.ErrTrezorPINNeeded.Error()) { + val, err = b.readPinAndReopenWallet(call) + if err == nil { + return val + } + } + // Check if the user needs to input a passphrase + if !strings.HasSuffix(err.Error(), usbwallet.ErrTrezorPassphraseNeeded.Error()) { throwJSException(err.Error()) } + val, err = b.readPassphraseAndReopenWallet(call) + if err != nil { + throwJSException(err.Error()) + } + return val +} + +func (b *bridge) readPassphraseAndReopenWallet(call otto.FunctionCall) (otto.Value, error) { + var passwd otto.Value + wallet := call.Argument(0) + if input, err := b.prompter.PromptPassword("Please enter your passphrase: "); err != nil { + throwJSException(err.Error()) + } else { + passwd, _ = otto.ToValue(input) + } + return call.Otto.Call("jeth.openWallet", nil, wallet, passwd) +} + +func (b *bridge) readPinAndReopenWallet(call otto.FunctionCall) (otto.Value, error) { + var passwd otto.Value + wallet := call.Argument(0) // Trezor PIN matrix input requested, display the matrix to the user and fetch the data fmt.Fprintf(b.printer, "Look at the device for number positions\n\n") fmt.Fprintf(b.printer, "7 | 8 | 9\n") @@ -121,10 +149,7 @@ func (b *bridge) OpenWallet(call otto.FunctionCall) (response otto.Value) { } else { passwd, _ = otto.ToValue(input) } - if val, err = call.Otto.Call("jeth.openWallet", nil, wallet, passwd); err != nil { - throwJSException(err.Error()) - } - return val + return call.Otto.Call("jeth.openWallet", nil, wallet, passwd) } // UnlockAccount is a wrapper around the personal.unlockAccount RPC method that diff --git a/core/blockchain.go b/core/blockchain.go index 49aedf6693..156efe303a 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -108,7 +108,6 @@ type BlockChain struct { scope event.SubscriptionScope genesisBlock *types.Block - mu sync.RWMutex // global mutex for locking chain operations chainmu sync.RWMutex // blockchain insertion lock procmu sync.RWMutex // block processor lock @@ -281,8 +280,8 @@ func (bc *BlockChain) loadLastState() error { func (bc *BlockChain) SetHead(head uint64) error { log.Warn("Rewinding blockchain", "target", head) - bc.mu.Lock() - defer bc.mu.Unlock() + bc.chainmu.Lock() + defer bc.chainmu.Unlock() // Rewind the header chain, deleting all block bodies until then delFn := func(db rawdb.DatabaseDeleter, hash common.Hash, num uint64) { @@ -340,9 +339,9 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { return err } // If all checks out, manually set the head block - bc.mu.Lock() + bc.chainmu.Lock() bc.currentBlock.Store(block) - bc.mu.Unlock() + bc.chainmu.Unlock() log.Info("Committed new head block", "number", block.Number(), "hash", hash) return nil @@ -420,8 +419,8 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { if err := bc.SetHead(0); err != nil { return err } - bc.mu.Lock() - defer bc.mu.Unlock() + bc.chainmu.Lock() + defer bc.chainmu.Unlock() // Prepare the genesis block and reinitialise the chain if err := bc.hc.WriteTd(genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil { @@ -468,8 +467,8 @@ func (bc *BlockChain) Export(w io.Writer) error { // ExportN writes a subset of the active chain to the given writer. func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { - bc.mu.RLock() - defer bc.mu.RUnlock() + bc.chainmu.RLock() + defer bc.chainmu.RUnlock() if first > last { return fmt.Errorf("export failed: first (%d) is greater than last (%d)", first, last) @@ -490,7 +489,6 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { reported = time.Now() } } - return nil } @@ -646,6 +644,9 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts { return nil } receipts := rawdb.ReadReceipts(bc.db, hash, *number) + if receipts == nil { + return nil + } bc.receiptsCache.Add(hash, receipts) return receipts } @@ -756,8 +757,8 @@ const ( // Rollback is designed to remove a chain of links from the database that aren't // certain enough to be valid. func (bc *BlockChain) Rollback(chain []common.Hash) { - bc.mu.Lock() - defer bc.mu.Unlock() + bc.chainmu.Lock() + defer bc.chainmu.Unlock() for i := len(chain) - 1; i >= 0; i-- { hash := chain[i] @@ -881,7 +882,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } // Update the head fast sync block if better - bc.mu.Lock() + bc.chainmu.Lock() head := blockChain[len(blockChain)-1] if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case currentFastBlock := bc.CurrentFastBlock() @@ -890,7 +891,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ bc.currentFastBlock.Store(head) } } - bc.mu.Unlock() + bc.chainmu.Unlock() context := []interface{}{ "count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)), @@ -924,6 +925,15 @@ func (bc *BlockChain) WriteBlockWithoutState(block *types.Block, td *big.Int) (e // WriteBlockWithState writes the block and all associated state to the database. func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) { + bc.chainmu.Lock() + defer bc.chainmu.Unlock() + + return bc.writeBlockWithState(block, receipts, state) +} + +// writeBlockWithState writes the block and all associated state to the database, +// but is expects the chain mutex to be held. +func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) { bc.wg.Add(1) defer bc.wg.Done() @@ -933,9 +943,6 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. return NonStatTy, consensus.ErrUnknownAncestor } // Make sure no inconsistent state is leaked during insertion - bc.mu.Lock() - defer bc.mu.Unlock() - currentBlock := bc.CurrentBlock() localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) externTd := new(big.Int).Add(block.Difficulty(), ptd) @@ -1212,7 +1219,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] proctime := time.Since(start) // Write the block to the chain and get the status. - status, err := bc.WriteBlockWithState(block, receipts, state) + status, err := bc.writeBlockWithState(block, receipts, state) t3 := time.Now() if err != nil { return it.index, events, coalescedLogs, err @@ -1281,7 +1288,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, []*types.Log, error) { var ( externTd *big.Int - current = bc.CurrentBlock().NumberU64() + current = bc.CurrentBlock() ) // The first sidechain block error is already verified to be ErrPrunedAncestor. // Since we don't import them here, we expect ErrUnknownAncestor for the remaining @@ -1290,7 +1297,7 @@ func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, [ block, err := it.current(), consensus.ErrPrunedAncestor for ; block != nil && (err == consensus.ErrPrunedAncestor); block, err = it.next() { // Check the canonical state root for that number - if number := block.NumberU64(); current >= number { + if number := block.NumberU64(); current.NumberU64() >= number { if canonical := bc.GetBlockByNumber(number); canonical != nil && canonical.Root() == block.Root() { // This is most likely a shadow-state attack. When a fork is imported into the // database, and it eventually reaches a block height which is not pruned, we @@ -1329,7 +1336,7 @@ func (bc *BlockChain) insertSidechain(it *insertIterator) (int, []interface{}, [ // // If the externTd was larger than our local TD, we now need to reimport the previous // blocks to regenerate the required state - localTd := bc.GetTd(bc.CurrentBlock().Hash(), current) + localTd := bc.GetTd(current.Hash(), current.NumberU64()) if localTd.Cmp(externTd) > 0 { log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().NumberU64(), "sidetd", externTd, "localtd", localTd) return it.index, nil, nil, err @@ -1597,36 +1604,12 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (i defer bc.wg.Done() whFunc := func(header *types.Header) error { - bc.mu.Lock() - defer bc.mu.Unlock() - _, err := bc.hc.WriteHeader(header) return err } - return bc.hc.InsertHeaderChain(chain, whFunc, start) } -// writeHeader writes a header into the local chain, given that its parent is -// already known. If the total difficulty of the newly inserted header becomes -// greater than the current known TD, the canonical chain is re-routed. -// -// Note: This method is not concurrent-safe with inserting blocks simultaneously -// into the chain, as side effects caused by reorganisations cannot be emulated -// without the real blocks. Hence, writing headers directly should only be done -// in two scenarios: pure-header mode of operation (light clients), or properly -// separated header/block phases (non-archive clients). -func (bc *BlockChain) writeHeader(header *types.Header) error { - bc.wg.Add(1) - defer bc.wg.Done() - - bc.mu.Lock() - defer bc.mu.Unlock() - - _, err := bc.hc.WriteHeader(header) - return err -} - // CurrentHeader retrieves the current head header of the canonical chain. The // header is retrieved from the HeaderChain's internal cache. func (bc *BlockChain) CurrentHeader() *types.Header { @@ -1675,8 +1658,8 @@ func (bc *BlockChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []com // // Note: ancestor == 0 returns the same block, 1 returns its parent and so on. func (bc *BlockChain) GetAncestor(hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) { - bc.chainmu.Lock() - defer bc.chainmu.Unlock() + bc.chainmu.RLock() + defer bc.chainmu.RUnlock() return bc.hc.GetAncestor(hash, number, ancestor, maxNonCanonical) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 5ab29e205f..cfc44bf794 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -162,11 +162,11 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { blockchain.reportBlock(block, receipts, err) return err } - blockchain.mu.Lock() + blockchain.chainmu.Lock() rawdb.WriteTd(blockchain.db, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTdByHash(block.ParentHash()))) rawdb.WriteBlock(blockchain.db, block) statedb.Commit(false) - blockchain.mu.Unlock() + blockchain.chainmu.Unlock() } return nil } @@ -180,10 +180,10 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error return err } // Manually insert the header into the database, but don't reorganise (allows subsequent testing) - blockchain.mu.Lock() + blockchain.chainmu.Lock() rawdb.WriteTd(blockchain.db, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTdByHash(header.ParentHash))) rawdb.WriteHeader(blockchain.db, header) - blockchain.mu.Unlock() + blockchain.chainmu.Unlock() } return nil } diff --git a/core/genesis.go b/core/genesis.go index c96cb17a36..62cde87b58 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -183,6 +183,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, constant newcfg := genesis.configOrDefault(stored) if constantinopleOverride != nil { newcfg.ConstantinopleBlock = constantinopleOverride + newcfg.PetersburgBlock = constantinopleOverride } storedcfg := rawdb.ReadChainConfig(db, stored) if storedcfg == nil { diff --git a/core/headerchain.go b/core/headerchain.go index d2093113c0..8904dd887b 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -219,14 +219,18 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header, checkFreq int) // Generate the list of seal verification requests, and start the parallel verifier seals := make([]bool, len(chain)) - for i := 0; i < len(seals)/checkFreq; i++ { - index := i*checkFreq + hc.rand.Intn(checkFreq) - if index >= len(seals) { - index = len(seals) - 1 + if checkFreq != 0 { + // In case of checkFreq == 0 all seals are left false. + for i := 0; i < len(seals)/checkFreq; i++ { + index := i*checkFreq + hc.rand.Intn(checkFreq) + if index >= len(seals) { + index = len(seals) - 1 + } + seals[index] = true } - seals[index] = true + // Last should always be verified to avoid junk. + seals[len(seals)-1] = true } - seals[len(seals)-1] = true // Last should always be verified to avoid junk abort, results := hc.engine.VerifyHeaders(hc, chain, seals) defer close(abort) diff --git a/core/types/block.go b/core/types/block.go index 57905d8c7a..b84e13c65a 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -21,10 +21,10 @@ import ( "encoding/binary" "io" "math/big" + "reflect" "sort" "sync/atomic" "time" - "unsafe" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -102,10 +102,12 @@ func (h *Header) Hash() common.Hash { return rlpHash(h) } +var headerSize = common.StorageSize(reflect.TypeOf(Header{}).Size()) + // Size returns the approximate memory used by all internal contents. It is used // to approximate and limit the memory consumption of various caches. func (h *Header) Size() common.StorageSize { - return common.StorageSize(unsafe.Sizeof(*h)) + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen()+h.Time.BitLen())/8) + return headerSize + common.StorageSize(len(h.Extra)+(h.Difficulty.BitLen()+h.Number.BitLen()+h.Time.BitLen())/8) } func rlpHash(x interface{}) (h common.Hash) { diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index df79f86eca..f22463c331 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// memoryGasCosts calculates the quadratic gas for memory expansion. It does so +// memoryGasCost calculates the quadratic gas for memory expansion. It does so // only for the memory region that is expanded, not the total memory. func memoryGasCost(mem *Memory, newMemSize uint64) (uint64, error) { @@ -121,7 +121,9 @@ func gasSStore(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, m current = evm.StateDB.GetState(contract.Address(), common.BigToHash(x)) ) // The legacy gas metering only takes into consideration the current state - if !evm.chainRules.IsConstantinople { + // Legacy rules should be applied if we are in Petersburg (removal of EIP-1283) + // OR Constantinople is not active + if evm.chainRules.IsPetersburg || !evm.chainRules.IsConstantinople { // This checks for 3 scenario's and calculates gas accordingly: // // 1. From a zero-value address to a non-zero value (NEW VALUE) @@ -380,7 +382,7 @@ func gasExp(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem gas = expByteLen * gt.ExpByte // no overflow check required. Max is 256 * ExpByte gas overflow bool ) - if gas, overflow = math.SafeAdd(gas, GasSlowStep); overflow { + if gas, overflow = math.SafeAdd(gas, params.ExpGas); overflow { return 0, errGasUintOverflow } return gas, nil diff --git a/core/vm/logger_json.go b/core/vm/logger_json.go index ac3c407597..ff379a4efd 100644 --- a/core/vm/logger_json.go +++ b/core/vm/logger_json.go @@ -34,7 +34,11 @@ type JSONLogger struct { // NewJSONLogger creates a new EVM tracer that prints execution steps as JSON objects // into the provided stream. func NewJSONLogger(cfg *LogConfig, writer io.Writer) *JSONLogger { - return &JSONLogger{json.NewEncoder(writer), cfg} + l := &JSONLogger{json.NewEncoder(writer), cfg} + if l.cfg == nil { + l.cfg = &LogConfig{} + } + return l } func (l *JSONLogger) CaptureStart(from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) error { diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 0b8f8aa00b..5096150c3f 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -576,6 +576,7 @@ func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block vmConf vm.Config dump *os.File + writer *bufio.Writer err error ) // If the transaction needs tracing, swap out the configs @@ -590,16 +591,19 @@ func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block dumps = append(dumps, dump.Name()) // Swap out the noop logger to the standard tracer + writer = bufio.NewWriter(dump) vmConf = vm.Config{ Debug: true, - Tracer: vm.NewJSONLogger(&logConfig, bufio.NewWriter(dump)), + Tracer: vm.NewJSONLogger(&logConfig, writer), EnablePreimageRecording: true, } } // Execute the transaction and flush any traces to disk vmenv := vm.NewEVM(vmctx, statedb, api.config, vmConf) _, _, _, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) - + if writer != nil { + writer.Flush() + } if dump != nil { dump.Close() log.Info("Wrote standard trace", "file", dump.Name()) diff --git a/eth/config.go b/eth/config.go index 7c041d1af7..f71b8dfee4 100644 --- a/eth/config.go +++ b/eth/config.go @@ -91,8 +91,12 @@ type Config struct { Whitelist map[uint64]common.Hash `toml:"-"` // Light client options - LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests - LightPeers int `toml:",omitempty"` // Maximum number of LES client peers + LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests + LightPeers int `toml:",omitempty"` // Maximum number of LES client peers + OnlyAnnounce bool // Maximum number of LES client peers + + // Ultra Light client options + ULC *ULCConfig `toml:",omitempty"` // Database options SkipBcVersionCheck bool `toml:"-"` diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index f0b5e8064b..94f05f9674 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -110,7 +110,6 @@ type Fetcher struct { notify chan *announce inject chan *inject - blockFilter chan chan []*types.Block headerFilter chan chan *headerFilterTask bodyFilter chan chan *bodyFilterTask @@ -150,7 +149,6 @@ func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBloc return &Fetcher{ notify: make(chan *announce), inject: make(chan *inject), - blockFilter: make(chan chan []*types.Block), headerFilter: make(chan chan *headerFilterTask), bodyFilter: make(chan chan *bodyFilterTask), done: make(chan common.Hash), diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/fetcher_test.go index 3d4f0d1e59..a86e773e3b 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/fetcher_test.go @@ -244,7 +244,7 @@ func verifyImportEvent(t *testing.T, imported chan *types.Block, arrive bool) { select { case <-imported: t.Fatalf("import invoked") - case <-time.After(10 * time.Millisecond): + case <-time.After(20 * time.Millisecond): } } } diff --git a/eth/gen_config.go b/eth/gen_config.go index 2777aa9e83..e05b963abe 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -23,10 +23,12 @@ func (c Config) MarshalTOML() (interface{}, error) { NetworkId uint64 SyncMode downloader.SyncMode NoPruning bool - LightServ int `toml:",omitempty"` - LightPeers int `toml:",omitempty"` - SkipBcVersionCheck bool `toml:"-"` - DatabaseHandles int `toml:"-"` + LightServ int `toml:",omitempty"` + LightPeers int `toml:",omitempty"` + OnlyAnnounce bool + ULC *ULCConfig `toml:",omitempty"` + SkipBcVersionCheck bool `toml:"-"` + DatabaseHandles int `toml:"-"` DatabaseCache int TrieCleanCache int TrieDirtyCache int @@ -54,6 +56,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.NoPruning = c.NoPruning enc.LightServ = c.LightServ enc.LightPeers = c.LightPeers + enc.OnlyAnnounce = c.OnlyAnnounce + enc.ULC = c.ULC enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache @@ -71,6 +75,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.Ethash = c.Ethash enc.TxPool = c.TxPool enc.GPO = c.GPO + enc.EnablePreimageRecording = c.EnablePreimageRecording enc.DocRoot = c.DocRoot enc.EWASMInterpreter = c.EWASMInterpreter @@ -85,10 +90,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { NetworkId *uint64 SyncMode *downloader.SyncMode NoPruning *bool - LightServ *int `toml:",omitempty"` - LightPeers *int `toml:",omitempty"` - SkipBcVersionCheck *bool `toml:"-"` - DatabaseHandles *int `toml:"-"` + LightServ *int `toml:",omitempty"` + LightPeers *int `toml:",omitempty"` + OnlyAnnounce *bool + ULC *ULCConfig `toml:",omitempty"` + SkipBcVersionCheck *bool `toml:"-"` + DatabaseHandles *int `toml:"-"` DatabaseCache *int TrieCleanCache *int TrieDirtyCache *int @@ -131,6 +138,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.LightPeers != nil { c.LightPeers = *dec.LightPeers } + if dec.OnlyAnnounce != nil { + c.OnlyAnnounce = *dec.OnlyAnnounce + } + if dec.ULC != nil { + c.ULC = dec.ULC + } if dec.SkipBcVersionCheck != nil { c.SkipBcVersionCheck = *dec.SkipBcVersionCheck } diff --git a/eth/ulc_config.go b/eth/ulc_config.go new file mode 100644 index 0000000000..effa6da217 --- /dev/null +++ b/eth/ulc_config.go @@ -0,0 +1,9 @@ +package eth + +const DefaultULCMinTrustedFraction = 75 + +// ULCConfig is a Ultra Light client options. +type ULCConfig struct { + TrustedServers []string `toml:",omitempty"` // A list of trusted servers + MinTrustedFraction int `toml:",omitempty"` // Minimum percentage of connected trusted servers to validate trusted (1-100) +} diff --git a/graphql/grahpql.go b/graphql/grahpql.go new file mode 100644 index 0000000000..1eca789567 --- /dev/null +++ b/graphql/grahpql.go @@ -0,0 +1,1104 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package graphql provides a GraphQL interface to Ethereum node data. +package graphql + +import ( + "context" + "fmt" + "net" + "net/http" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/filters" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" + graphqlgo "github.com/graph-gophers/graphql-go" + "github.com/graph-gophers/graphql-go/relay" +) + +// Account represents an Ethereum account at a particular block. +type Account struct { + backend *eth.EthAPIBackend + address common.Address + blockNumber rpc.BlockNumber +} + +// getState fetches the StateDB object for an account. +func (a *Account) getState(ctx context.Context) (*state.StateDB, error) { + state, _, err := a.backend.StateAndHeaderByNumber(ctx, a.blockNumber) + return state, err +} + +func (a *Account) Address(ctx context.Context) (common.Address, error) { + return a.address, nil +} + +func (a *Account) Balance(ctx context.Context) (hexutil.Big, error) { + state, err := a.getState(ctx) + if err != nil { + return hexutil.Big{}, err + } + + return hexutil.Big(*state.GetBalance(a.address)), nil +} + +func (a *Account) TransactionCount(ctx context.Context) (hexutil.Uint64, error) { + state, err := a.getState(ctx) + if err != nil { + return 0, err + } + + return hexutil.Uint64(state.GetNonce(a.address)), nil +} + +func (a *Account) Code(ctx context.Context) (hexutil.Bytes, error) { + state, err := a.getState(ctx) + if err != nil { + return hexutil.Bytes{}, err + } + + return hexutil.Bytes(state.GetCode(a.address)), nil +} + +func (a *Account) Storage(ctx context.Context, args struct{ Slot common.Hash }) (common.Hash, error) { + state, err := a.getState(ctx) + if err != nil { + return common.Hash{}, err + } + + return state.GetState(a.address, args.Slot), nil +} + +// Log represents an individual log message. All arguments are mandatory. +type Log struct { + backend *eth.EthAPIBackend + transaction *Transaction + log *types.Log +} + +func (l *Log) Transaction(ctx context.Context) *Transaction { + return l.transaction +} + +func (l *Log) Account(ctx context.Context, args BlockNumberArgs) *Account { + return &Account{ + backend: l.backend, + address: l.log.Address, + blockNumber: args.Number(), + } +} + +func (l *Log) Index(ctx context.Context) int32 { + return int32(l.log.Index) +} + +func (l *Log) Topics(ctx context.Context) []common.Hash { + return l.log.Topics +} + +func (l *Log) Data(ctx context.Context) hexutil.Bytes { + return hexutil.Bytes(l.log.Data) +} + +// Transactionn represents an Ethereum transaction. +// backend and hash are mandatory; all others will be fetched when required. +type Transaction struct { + backend *eth.EthAPIBackend + hash common.Hash + tx *types.Transaction + block *Block + index uint64 +} + +// resolve returns the internal transaction object, fetching it if needed. +func (t *Transaction) resolve(ctx context.Context) (*types.Transaction, error) { + if t.tx == nil { + tx, blockHash, _, index := rawdb.ReadTransaction(t.backend.ChainDb(), t.hash) + if tx != nil { + t.tx = tx + t.block = &Block{ + backend: t.backend, + hash: blockHash, + } + t.index = index + } else { + t.tx = t.backend.GetPoolTransaction(t.hash) + } + } + return t.tx, nil +} + +func (tx *Transaction) Hash(ctx context.Context) common.Hash { + return tx.hash +} + +func (t *Transaction) InputData(ctx context.Context) (hexutil.Bytes, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return hexutil.Bytes{}, err + } + return hexutil.Bytes(tx.Data()), nil +} + +func (t *Transaction) Gas(ctx context.Context) (hexutil.Uint64, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return 0, err + } + return hexutil.Uint64(tx.Gas()), nil +} + +func (t *Transaction) GasPrice(ctx context.Context) (hexutil.Big, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return hexutil.Big{}, err + } + return hexutil.Big(*tx.GasPrice()), nil +} + +func (t *Transaction) Value(ctx context.Context) (hexutil.Big, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return hexutil.Big{}, err + } + return hexutil.Big(*tx.Value()), nil +} + +func (t *Transaction) Nonce(ctx context.Context) (hexutil.Uint64, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return 0, err + } + return hexutil.Uint64(tx.Nonce()), nil +} + +func (t *Transaction) To(ctx context.Context, args BlockNumberArgs) (*Account, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return nil, err + } + + to := tx.To() + if to == nil { + return nil, nil + } + + return &Account{ + backend: t.backend, + address: *to, + blockNumber: args.Number(), + }, nil +} + +func (t *Transaction) From(ctx context.Context, args BlockNumberArgs) (*Account, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return nil, err + } + + var signer types.Signer = types.FrontierSigner{} + if tx.Protected() { + signer = types.NewEIP155Signer(tx.ChainId()) + } + from, _ := types.Sender(signer, tx) + + return &Account{ + backend: t.backend, + address: from, + blockNumber: args.Number(), + }, nil +} + +func (t *Transaction) Block(ctx context.Context) (*Block, error) { + if _, err := t.resolve(ctx); err != nil { + return nil, err + } + return t.block, nil +} + +func (t *Transaction) Index(ctx context.Context) (*int32, error) { + if _, err := t.resolve(ctx); err != nil { + return nil, err + } + if t.block == nil { + return nil, nil + } + index := int32(t.index) + return &index, nil +} + +// getReceipt returns the receipt associated with this transaction, if any. +func (t *Transaction) getReceipt(ctx context.Context) (*types.Receipt, error) { + if _, err := t.resolve(ctx); err != nil { + return nil, err + } + + if t.block == nil { + return nil, nil + } + + receipts, err := t.block.resolveReceipts(ctx) + if err != nil { + return nil, err + } + + return receipts[t.index], nil +} + +func (t *Transaction) Status(ctx context.Context) (*hexutil.Uint64, error) { + receipt, err := t.getReceipt(ctx) + if err != nil || receipt == nil { + return nil, err + } + + ret := hexutil.Uint64(receipt.Status) + return &ret, nil +} + +func (t *Transaction) GasUsed(ctx context.Context) (*hexutil.Uint64, error) { + receipt, err := t.getReceipt(ctx) + if err != nil || receipt == nil { + return nil, err + } + + ret := hexutil.Uint64(receipt.GasUsed) + return &ret, nil +} + +func (t *Transaction) CumulativeGasUsed(ctx context.Context) (*hexutil.Uint64, error) { + receipt, err := t.getReceipt(ctx) + if err != nil || receipt == nil { + return nil, err + } + + ret := hexutil.Uint64(receipt.CumulativeGasUsed) + return &ret, nil +} + +func (t *Transaction) CreatedContract(ctx context.Context, args BlockNumberArgs) (*Account, error) { + receipt, err := t.getReceipt(ctx) + if err != nil || receipt == nil || receipt.ContractAddress == (common.Address{}) { + return nil, err + } + + return &Account{ + backend: t.backend, + address: receipt.ContractAddress, + blockNumber: args.Number(), + }, nil +} + +func (t *Transaction) Logs(ctx context.Context) (*[]*Log, error) { + receipt, err := t.getReceipt(ctx) + if err != nil || receipt == nil { + return nil, err + } + + ret := make([]*Log, 0, len(receipt.Logs)) + for _, log := range receipt.Logs { + ret = append(ret, &Log{ + backend: t.backend, + transaction: t, + log: log, + }) + } + return &ret, nil +} + +// Block represennts an Ethereum block. +// backend, and either num or hash are mandatory. All other fields are lazily fetched +// when required. +type Block struct { + backend *eth.EthAPIBackend + num *rpc.BlockNumber + hash common.Hash + header *types.Header + block *types.Block + receipts []*types.Receipt +} + +// resolve returns the internal Block object representing this block, fetching +// it if necessary. +func (b *Block) resolve(ctx context.Context) (*types.Block, error) { + if b.block != nil { + return b.block, nil + } + + var err error + if b.hash != (common.Hash{}) { + b.block, err = b.backend.GetBlock(ctx, b.hash) + } else { + b.block, err = b.backend.BlockByNumber(ctx, *b.num) + } + if b.block != nil { + b.header = b.block.Header() + } + return b.block, err +} + +// resolveHeader returns the internal Header object for this block, fetching it +// if necessary. Call this function instead of `resolve` unless you need the +// additional data (transactions and uncles). +func (b *Block) resolveHeader(ctx context.Context) (*types.Header, error) { + if b.header == nil { + if _, err := b.resolve(ctx); err != nil { + return nil, err + } + } + return b.header, nil +} + +// resolveReceipts returns the list of receipts for this block, fetching them +// if necessary. +func (b *Block) resolveReceipts(ctx context.Context) ([]*types.Receipt, error) { + if b.receipts == nil { + hash := b.hash + if hash == (common.Hash{}) { + header, err := b.resolveHeader(ctx) + if err != nil { + return nil, err + } + hash = header.Hash() + } + + receipts, err := b.backend.GetReceipts(ctx, hash) + if err != nil { + return nil, err + } + b.receipts = []*types.Receipt(receipts) + } + return b.receipts, nil +} + +func (b *Block) Number(ctx context.Context) (hexutil.Uint64, error) { + if b.num == nil || *b.num == rpc.LatestBlockNumber { + header, err := b.resolveHeader(ctx) + if err != nil { + return 0, err + } + num := rpc.BlockNumber(header.Number.Uint64()) + b.num = &num + } + return hexutil.Uint64(*b.num), nil +} + +func (b *Block) Hash(ctx context.Context) (common.Hash, error) { + if b.hash == (common.Hash{}) { + header, err := b.resolveHeader(ctx) + if err != nil { + return common.Hash{}, err + } + b.hash = header.Hash() + } + return b.hash, nil +} + +func (b *Block) GasLimit(ctx context.Context) (hexutil.Uint64, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return 0, err + } + return hexutil.Uint64(header.GasLimit), nil +} + +func (b *Block) GasUsed(ctx context.Context) (hexutil.Uint64, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return 0, err + } + return hexutil.Uint64(header.GasUsed), nil +} + +func (b *Block) Parent(ctx context.Context) (*Block, error) { + // If the block hasn't been fetched, and we'll need it, fetch it. + if b.num == nil && b.hash != (common.Hash{}) && b.header == nil { + if _, err := b.resolve(ctx); err != nil { + return nil, err + } + } + + if b.header != nil && b.block.NumberU64() > 0 { + num := rpc.BlockNumber(b.header.Number.Uint64() - 1) + return &Block{ + backend: b.backend, + num: &num, + hash: b.header.ParentHash, + }, nil + } else if b.num != nil && *b.num != 0 { + num := *b.num - 1 + return &Block{ + backend: b.backend, + num: &num, + }, nil + } + return nil, nil +} + +func (b *Block) Difficulty(ctx context.Context) (hexutil.Big, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return hexutil.Big{}, err + } + return hexutil.Big(*header.Difficulty), nil +} + +func (b *Block) Timestamp(ctx context.Context) (hexutil.Big, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return hexutil.Big{}, err + } + return hexutil.Big(*header.Time), nil +} + +func (b *Block) Nonce(ctx context.Context) (hexutil.Bytes, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return hexutil.Bytes{}, err + } + return hexutil.Bytes(header.Nonce[:]), nil +} + +func (b *Block) MixHash(ctx context.Context) (common.Hash, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return common.Hash{}, err + } + return header.MixDigest, nil +} + +func (b *Block) TransactionsRoot(ctx context.Context) (common.Hash, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return common.Hash{}, err + } + return header.TxHash, nil +} + +func (b *Block) StateRoot(ctx context.Context) (common.Hash, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return common.Hash{}, err + } + return header.Root, nil +} + +func (b *Block) ReceiptsRoot(ctx context.Context) (common.Hash, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return common.Hash{}, err + } + return header.ReceiptHash, nil +} + +func (b *Block) OmmerHash(ctx context.Context) (common.Hash, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return common.Hash{}, err + } + return header.UncleHash, nil +} + +func (b *Block) OmmerCount(ctx context.Context) (*int32, error) { + block, err := b.resolve(ctx) + if err != nil || block == nil { + return nil, err + } + count := int32(len(block.Uncles())) + return &count, err +} + +func (b *Block) Ommers(ctx context.Context) (*[]*Block, error) { + block, err := b.resolve(ctx) + if err != nil || block == nil { + return nil, err + } + + ret := make([]*Block, 0, len(block.Uncles())) + for _, uncle := range block.Uncles() { + blockNumber := rpc.BlockNumber(uncle.Number.Uint64()) + ret = append(ret, &Block{ + backend: b.backend, + num: &blockNumber, + hash: uncle.Hash(), + header: uncle, + }) + } + return &ret, nil +} + +func (b *Block) ExtraData(ctx context.Context) (hexutil.Bytes, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return hexutil.Bytes{}, err + } + return hexutil.Bytes(header.Extra), nil +} + +func (b *Block) LogsBloom(ctx context.Context) (hexutil.Bytes, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return hexutil.Bytes{}, err + } + return hexutil.Bytes(header.Bloom.Bytes()), nil +} + +func (b *Block) TotalDifficulty(ctx context.Context) (hexutil.Big, error) { + h := b.hash + if h == (common.Hash{}) { + header, err := b.resolveHeader(ctx) + if err != nil { + return hexutil.Big{}, err + } + h = header.Hash() + } + + return hexutil.Big(*b.backend.GetTd(h)), nil +} + +// BlockNumberArgs encapsulates arguments to accessors that specify a block number. +type BlockNumberArgs struct { + Block *hexutil.Uint64 +} + +// Number returns the provided block number, or rpc.LatestBlockNumber if none +// was provided. +func (a BlockNumberArgs) Number() rpc.BlockNumber { + if a.Block != nil { + return rpc.BlockNumber(*a.Block) + } + return rpc.LatestBlockNumber +} + +func (b *Block) Miner(ctx context.Context, args BlockNumberArgs) (*Account, error) { + block, err := b.resolve(ctx) + if err != nil { + return nil, err + } + + return &Account{ + backend: b.backend, + address: block.Coinbase(), + blockNumber: args.Number(), + }, nil +} + +func (b *Block) TransactionCount(ctx context.Context) (*int32, error) { + block, err := b.resolve(ctx) + if err != nil || block == nil { + return nil, err + } + count := int32(len(block.Transactions())) + return &count, err +} + +func (b *Block) Transactions(ctx context.Context) (*[]*Transaction, error) { + block, err := b.resolve(ctx) + if err != nil || block == nil { + return nil, err + } + + ret := make([]*Transaction, 0, len(block.Transactions())) + for i, tx := range block.Transactions() { + ret = append(ret, &Transaction{ + backend: b.backend, + hash: tx.Hash(), + tx: tx, + block: b, + index: uint64(i), + }) + } + return &ret, nil +} + +func (b *Block) TransactionAt(ctx context.Context, args struct{ Index int32 }) (*Transaction, error) { + block, err := b.resolve(ctx) + if err != nil || block == nil { + return nil, err + } + + txes := block.Transactions() + if args.Index < 0 || int(args.Index) >= len(txes) { + return nil, nil + } + + tx := txes[args.Index] + return &Transaction{ + backend: b.backend, + hash: tx.Hash(), + tx: tx, + block: b, + index: uint64(args.Index), + }, nil +} + +func (b *Block) OmmerAt(ctx context.Context, args struct{ Index int32 }) (*Block, error) { + block, err := b.resolve(ctx) + if err != nil || block == nil { + return nil, err + } + + uncles := block.Uncles() + if args.Index < 0 || int(args.Index) >= len(uncles) { + return nil, nil + } + + uncle := uncles[args.Index] + blockNumber := rpc.BlockNumber(uncle.Number.Uint64()) + return &Block{ + backend: b.backend, + num: &blockNumber, + hash: uncle.Hash(), + header: uncle, + }, nil +} + +// BlockFilterCriteria encapsulates criteria passed to a `logs` accessor inside +// a block. +type BlockFilterCriteria struct { + Addresses *[]common.Address // restricts matches to events created by specific contracts + + // The Topic list restricts matches to particular event topics. Each event has a list + // of topics. Topics matches a prefix of that list. An empty element slice matches any + // topic. Non-empty elements represent an alternative that matches any of the + // contained topics. + // + // Examples: + // {} or nil matches any topic list + // {{A}} matches topic A in first position + // {{}, {B}} matches any topic in first position, B in second position + // {{A}, {B}} matches topic A in first position, B in second position + // {{A, B}}, {C, D}} matches topic (A OR B) in first position, (C OR D) in second position + Topics *[][]common.Hash +} + +// runFilter accepts a filter and executes it, returning all its results as +// `Log` objects. +func runFilter(ctx context.Context, be *eth.EthAPIBackend, filter *filters.Filter) ([]*Log, error) { + logs, err := filter.Logs(ctx) + if err != nil || logs == nil { + return nil, err + } + + ret := make([]*Log, 0, len(logs)) + for _, log := range logs { + ret = append(ret, &Log{ + backend: be, + transaction: &Transaction{backend: be, hash: log.TxHash}, + log: log, + }) + } + return ret, nil +} + +func (b *Block) Logs(ctx context.Context, args struct{ Filter BlockFilterCriteria }) ([]*Log, error) { + var addresses []common.Address + if args.Filter.Addresses != nil { + addresses = *args.Filter.Addresses + } + + var topics [][]common.Hash + if args.Filter.Topics != nil { + topics = *args.Filter.Topics + } + + hash := b.hash + if hash == (common.Hash{}) { + block, err := b.resolve(ctx) + if err != nil { + return nil, err + } + hash = block.Hash() + } + + // Construct the range filter + filter := filters.NewBlockFilter(b.backend, hash, addresses, topics) + + // Run the filter and return all the logs + return runFilter(ctx, b.backend, filter) +} + +// Resolver is the top-level object in the GraphQL hierarchy. +type Resolver struct { + backend *eth.EthAPIBackend +} + +func (r *Resolver) Block(ctx context.Context, args struct { + Number *hexutil.Uint64 + Hash *common.Hash +}) (*Block, error) { + var block *Block + if args.Number != nil { + num := rpc.BlockNumber(uint64(*args.Number)) + block = &Block{ + backend: r.backend, + num: &num, + } + } else if args.Hash != nil { + block = &Block{ + backend: r.backend, + hash: *args.Hash, + } + } else { + num := rpc.LatestBlockNumber + block = &Block{ + backend: r.backend, + num: &num, + } + } + + // Resolve the block; if it doesn't exist, return nil. + b, err := block.resolve(ctx) + if err != nil { + return nil, err + } else if b == nil { + return nil, nil + } + return block, nil +} + +func (r *Resolver) Blocks(ctx context.Context, args struct { + From hexutil.Uint64 + To *hexutil.Uint64 +}) ([]*Block, error) { + from := rpc.BlockNumber(args.From) + + var to rpc.BlockNumber + if args.To != nil { + to = rpc.BlockNumber(*args.To) + } else { + to = rpc.BlockNumber(r.backend.CurrentBlock().Number().Int64()) + } + + if to < from { + return []*Block{}, nil + } + + ret := make([]*Block, 0, to-from+1) + for i := from; i <= to; i++ { + num := i + ret = append(ret, &Block{ + backend: r.backend, + num: &num, + }) + } + return ret, nil +} + +func (r *Resolver) Account(ctx context.Context, args struct { + Address common.Address + BlockNumber *hexutil.Uint64 +}) *Account { + blockNumber := rpc.LatestBlockNumber + if args.BlockNumber != nil { + blockNumber = rpc.BlockNumber(*args.BlockNumber) + } + + return &Account{ + backend: r.backend, + address: args.Address, + blockNumber: blockNumber, + } +} + +func (r *Resolver) Transaction(ctx context.Context, args struct{ Hash common.Hash }) (*Transaction, error) { + tx := &Transaction{ + backend: r.backend, + hash: args.Hash, + } + + // Resolve the transaction; if it doesn't exist, return nil. + t, err := tx.resolve(ctx) + if err != nil { + return nil, err + } else if t == nil { + return nil, nil + } + return tx, nil +} + +func (r *Resolver) SendRawTransaction(ctx context.Context, args struct{ Data hexutil.Bytes }) (common.Hash, error) { + tx := new(types.Transaction) + if err := rlp.DecodeBytes(args.Data, tx); err != nil { + return common.Hash{}, err + } + hash, err := ethapi.SubmitTransaction(ctx, r.backend, tx) + return hash, err +} + +// CallData encapsulates arguments to `call` or `estimateGas`. +// All arguments are optional. +type CallData struct { + From *common.Address // The Ethereum address the call is from. + To *common.Address // The Ethereum address the call is to. + Gas *hexutil.Uint64 // The amount of gas provided for the call. + GasPrice *hexutil.Big // The price of each unit of gas, in wei. + Value *hexutil.Big // The value sent along with the call. + Data *hexutil.Bytes // Any data sent with the call. +} + +// CallResult encapsulates the result of an invocation of the `call` accessor. +type CallResult struct { + data hexutil.Bytes // The return data from the call + gasUsed hexutil.Uint64 // The amount of gas used + status hexutil.Uint64 // The return status of the call - 0 for failure or 1 for success. +} + +func (c *CallResult) Data() hexutil.Bytes { + return c.data +} + +func (c *CallResult) GasUsed() hexutil.Uint64 { + return c.gasUsed +} + +func (c *CallResult) Status() hexutil.Uint64 { + return c.status +} + +func (r *Resolver) Call(ctx context.Context, args struct { + Data ethapi.CallArgs + BlockNumber *hexutil.Uint64 +}) (*CallResult, error) { + blockNumber := rpc.LatestBlockNumber + if args.BlockNumber != nil { + blockNumber = rpc.BlockNumber(*args.BlockNumber) + } + + result, gas, failed, err := ethapi.DoCall(ctx, r.backend, args.Data, blockNumber, vm.Config{}, 5*time.Second) + status := hexutil.Uint64(1) + if failed { + status = 0 + } + return &CallResult{ + data: hexutil.Bytes(result), + gasUsed: hexutil.Uint64(gas), + status: status, + }, err +} + +func (r *Resolver) EstimateGas(ctx context.Context, args struct { + Data ethapi.CallArgs + BlockNumber *hexutil.Uint64 +}) (hexutil.Uint64, error) { + blockNumber := rpc.LatestBlockNumber + if args.BlockNumber != nil { + blockNumber = rpc.BlockNumber(*args.BlockNumber) + } + + gas, err := ethapi.DoEstimateGas(ctx, r.backend, args.Data, blockNumber) + return gas, err +} + +// FilterCritera encapsulates the arguments to `logs` on the root resolver object. +type FilterCriteria struct { + FromBlock *hexutil.Uint64 // beginning of the queried range, nil means genesis block + ToBlock *hexutil.Uint64 // end of the range, nil means latest block + Addresses *[]common.Address // restricts matches to events created by specific contracts + + // The Topic list restricts matches to particular event topics. Each event has a list + // of topics. Topics matches a prefix of that list. An empty element slice matches any + // topic. Non-empty elements represent an alternative that matches any of the + // contained topics. + // + // Examples: + // {} or nil matches any topic list + // {{A}} matches topic A in first position + // {{}, {B}} matches any topic in first position, B in second position + // {{A}, {B}} matches topic A in first position, B in second position + // {{A, B}}, {C, D}} matches topic (A OR B) in first position, (C OR D) in second position + Topics *[][]common.Hash +} + +func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria }) ([]*Log, error) { + // Convert the RPC block numbers into internal representations + begin := rpc.LatestBlockNumber.Int64() + if args.Filter.FromBlock != nil { + begin = int64(*args.Filter.FromBlock) + } + end := rpc.LatestBlockNumber.Int64() + if args.Filter.ToBlock != nil { + end = int64(*args.Filter.ToBlock) + } + + var addresses []common.Address + if args.Filter.Addresses != nil { + addresses = *args.Filter.Addresses + } + + var topics [][]common.Hash + if args.Filter.Topics != nil { + topics = *args.Filter.Topics + } + + // Construct the range filter + filter := filters.NewRangeFilter(filters.Backend(r.backend), begin, end, addresses, topics) + + return runFilter(ctx, r.backend, filter) +} + +func (r *Resolver) GasPrice(ctx context.Context) (hexutil.Big, error) { + price, err := r.backend.SuggestPrice(ctx) + return hexutil.Big(*price), err +} + +func (r *Resolver) ProtocolVersion(ctx context.Context) (int32, error) { + return int32(r.backend.ProtocolVersion()), nil +} + +// SyncState represents the synchronisation status returned from the `syncing` accessor. +type SyncState struct { + progress ethereum.SyncProgress +} + +func (s *SyncState) StartingBlock() hexutil.Uint64 { + return hexutil.Uint64(s.progress.StartingBlock) +} + +func (s *SyncState) CurrentBlock() hexutil.Uint64 { + return hexutil.Uint64(s.progress.CurrentBlock) +} + +func (s *SyncState) HighestBlock() hexutil.Uint64 { + return hexutil.Uint64(s.progress.HighestBlock) +} + +func (s *SyncState) PulledStates() *hexutil.Uint64 { + ret := hexutil.Uint64(s.progress.PulledStates) + return &ret +} + +func (s *SyncState) KnownStates() *hexutil.Uint64 { + ret := hexutil.Uint64(s.progress.KnownStates) + return &ret +} + +// Syncing returns false in case the node is currently not syncing with the network. It can be up to date or has not +// yet received the latest block headers from its pears. In case it is synchronizing: +// - startingBlock: block number this node started to synchronise from +// - currentBlock: block number this node is currently importing +// - highestBlock: block number of the highest block header this node has received from peers +// - pulledStates: number of state entries processed until now +// - knownStates: number of known state entries that still need to be pulled +func (r *Resolver) Syncing() (*SyncState, error) { + progress := r.backend.Downloader().Progress() + + // Return not syncing if the synchronisation already completed + if progress.CurrentBlock >= progress.HighestBlock { + return nil, nil + } + // Otherwise gather the block sync stats + return &SyncState{progress}, nil +} + +// NewHandler returns a new `http.Handler` that will answer GraphQL queries. +// It additionally exports an interactive query browser on the / endpoint. +func NewHandler(be *eth.EthAPIBackend) (http.Handler, error) { + q := Resolver{be} + + s, err := graphqlgo.ParseSchema(schema, &q) + if err != nil { + return nil, err + } + h := &relay.Handler{Schema: s} + + mux := http.NewServeMux() + mux.Handle("/", GraphiQL{}) + mux.Handle("/graphql", h) + mux.Handle("/graphql/", h) + return mux, nil +} + +// Service encapsulates a GraphQL service. +type Service struct { + endpoint string // The host:port endpoint for this service. + cors []string // Allowed CORS domains + vhosts []string // Recognised vhosts + timeouts rpc.HTTPTimeouts // Timeout settings for HTTP requests. + backend *eth.EthAPIBackend // The backend that queries will operate onn. + handler http.Handler // The `http.Handler` used to answer queries. + listener net.Listener // The listening socket. +} + +// Protocols returns the list of protocols exported by this service. +func (s *Service) Protocols() []p2p.Protocol { return nil } + +// APIs returns the list of APIs exported by this service. +func (s *Service) APIs() []rpc.API { return nil } + +// Start is called after all services have been constructed and the networking +// layer was also initialized to spawn any goroutines required by the service. +func (s *Service) Start(server *p2p.Server) error { + var err error + s.handler, err = NewHandler(s.backend) + if err != nil { + return err + } + + if s.listener, err = net.Listen("tcp", s.endpoint); err != nil { + return err + } + + go rpc.NewHTTPServer(s.cors, s.vhosts, s.timeouts, s.handler).Serve(s.listener) + log.Info("GraphQL endpoint opened", "url", fmt.Sprintf("http://%s", s.endpoint)) + return nil +} + +// Stop terminates all goroutines belonging to the service, blocking until they +// are all terminated. +func (s *Service) Stop() error { + if s.listener != nil { + s.listener.Close() + s.listener = nil + log.Info("GraphQL endpoint closed", "url", fmt.Sprintf("http://%s", s.endpoint)) + } + return nil +} + +// NewService constructs a new service instance. +func NewService(backend *eth.EthAPIBackend, endpoint string, cors, vhosts []string, timeouts rpc.HTTPTimeouts) (*Service, error) { + return &Service{ + endpoint: endpoint, + cors: cors, + vhosts: vhosts, + timeouts: timeouts, + backend: backend, + }, nil +} + +// RegisterGraphQLService is a utility function to construct a new service and register it against a node. +func RegisterGraphQLService(stack *node.Node, endpoint string, cors, vhosts []string, timeouts rpc.HTTPTimeouts) error { + return stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + var ethereum *eth.Ethereum + if err := ctx.Service(ðereum); err != nil { + return nil, err + } + return NewService(ethereum.APIBackend, endpoint, cors, vhosts, timeouts) + }) +} diff --git a/graphql/graphiql.go b/graphql/graphiql.go new file mode 100644 index 0000000000..6d9dda3e8c --- /dev/null +++ b/graphql/graphiql.go @@ -0,0 +1,95 @@ +// The MIT License (MIT) +// +// Copyright (c) 2016 Muhammed Thanish +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package graphql + +import ( + "bytes" + "fmt" + "net/http" +) + +// GraphiQL is an in-browser IDE for exploring GraphiQL APIs. +// This handler returns GraphiQL when requested. +// +// For more information, see https://github.com/graphql/graphiql. +type GraphiQL struct{} + +func respond(w http.ResponseWriter, body []byte, code int) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(code) + _, _ = w.Write(body) +} + +func errorJSON(msg string) []byte { + buf := bytes.Buffer{} + fmt.Fprintf(&buf, `{"error": "%s"}`, msg) + return buf.Bytes() +} + +func (h GraphiQL) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != "GET" { + respond(w, errorJSON("only GET requests are supported"), http.StatusMethodNotAllowed) + return + } + + w.Write(graphiql) +} + +var graphiql = []byte(` + + + + + + + + + + +
Loading...
+ + + +`) diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go new file mode 100644 index 0000000000..d63418398a --- /dev/null +++ b/graphql/graphql_test.go @@ -0,0 +1,29 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package graphql + +import ( + "testing" +) + +func TestBuildSchema(t *testing.T) { + // Make sure the schema can be parsed and matched up to the object model. + _, err := NewHandler(nil) + if err != nil { + t.Errorf("Could not construct GraphQL handler: %v", err) + } +} diff --git a/graphql/schema.go b/graphql/schema.go new file mode 100644 index 0000000000..c1ba87d2d6 --- /dev/null +++ b/graphql/schema.go @@ -0,0 +1,305 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package graphql + +const schema string = ` + # Bytes32 is a 32 byte binary string, represented as 0x-prefixed hexadecimal. + scalar Bytes32 + # Address is a 20 byte Ethereum address, represented as 0x-prefixed hexadecimal. + scalar Address + # Bytes is an arbitrary length binary string, represented as 0x-prefixed hexadecimal. + scalar Bytes + # BigInt is a large integer. Input is accepted as either a JSON number or as a string. + # Strings may be either decimal or 0x-prefixed hexadecimal. Output values are all + # 0x-prefixed hexadecimal. + scalar BigInt + # Long is a 64 bit unsigned integer. + scalar Long + + schema { + query: Query + mutation: Mutation + } + + # Account is an Ethereum account at a particular block. + type Account { + # Address is the address owning the account. + address: Address! + # Balance is the balance of the account, in wei. + balance: BigInt! + # TransactionCount is the number of transactions sent from this account, + # or in the case of a contract, the number of contracts created. Otherwise + # known as the nonce. + transactionCount: Long! + # Code contains the smart contract code for this account, if the account + # is a (non-self-destructed) contract. + code: Bytes! + # Storage provides access to the storage of a contract account, indexed + # by its 32 byte slot identifier. + storage(slot: Bytes32!): Bytes32! + } + + # Log is an Ethereum event log. + type Log { + # Index is the index of this log in the block. + index: Int! + # Account is the account which generated this log - this will always + # be a contract account. + account(block: Long): Account! + # Topics is a list of 0-4 indexed topics for the log. + topics: [Bytes32!]! + # Data is unindexed data for this log. + data: Bytes! + # Transaction is the transaction that generated this log entry. + transaction: Transaction! + } + + # Transaction is an Ethereum transaction. + type Transaction { + # Hash is the hash of this transaction. + hash: Bytes32! + # Nonce is the nonce of the account this transaction was generated with. + nonce: Long! + # Index is the index of this transaction in the parent block. This will + # be null if the transaction has not yet beenn mined. + index: Int + # From is the account that sent this transaction - this will always be + # an externally owned account. + from(block: Long): Account! + # To is the account the transaction was sent to. This is null for + # contract-creating transactions. + to(block: Long): Account + # Value is the value, in wei, sent along with this transaction. + value: BigInt! + # GasPrice is the price offered to miners for gas, in wei per unit. + gasPrice: BigInt! + # Gas is the maximum amount of gas this transaction can consume. + gas: Long! + # InputData is the data supplied to the target of the transaction. + inputData: Bytes! + # Block is the block this transaction was mined in. This will be null if + # the transaction has not yet been mined. + block: Block + + # Status is the return status of the transaction. This will be 1 if the + # transaction succeeded, or 0 if it failed (due to a revert, or due to + # running out of gas). If the transaction has not yet been mined, this + # field will be null. + status: Long + # GasUsed is the amount of gas that was used processing this transaction. + # If the transaction has not yet been mined, this field will be null. + gasUsed: Long + # CumulativeGasUsed is the total gas used in the block up to and including + # this transaction. If the transaction has not yet been mined, this field + # will be null. + cumulativeGasUsed: Long + # CreatedContract is the account that was created by a contract creation + # transaction. If the transaction was not a contract creation transaction, + # or it has not yet been mined, this field will be null. + createdContract(block: Long): Account + # Logs is a list of log entries emitted by this transaction. If the + # transaction has not yet been mined, this field will be null. + logs: [Log!] + } + + # BlockFilterCriteria encapsulates log filter criteria for a filter applied + # to a single block. + input BlockFilterCriteria { + # Addresses is list of addresses that are of interest. If this list is + # empty, results will not be filtered by address. + addresses: [Address!] + # Topics list restricts matches to particular event topics. Each event has a list + # of topics. Topics matches a prefix of that list. An empty element array matches any + # topic. Non-empty elements represent an alternative that matches any of the + # contained topics. + # + # Examples: + # - [] or nil matches any topic list + # - [[A]] matches topic A in first position + # - [[], [B]] matches any topic in first position, B in second position + # - [[A], [B]] matches topic A in first position, B in second position + # - [[A, B]], [C, D]] matches topic (A OR B) in first position, (C OR D) in second position + topics: [[Bytes32!]!] + } + + # Block is an Ethereum block. + type Block { + # Number is the number of this block, starting at 0 for the genesis block. + number: Long! + # Hash is the block hash of this block. + hash: Bytes32! + # Parent is the parent block of this block. + parent: Block + # Nonce is the block nonce, an 8 byte sequence determined by the miner. + nonce: Bytes! + # TransactionsRoot is the keccak256 hash of the root of the trie of transactions in this block. + transactionsRoot: Bytes32! + # TransactionCount is the number of transactions in this block. if + # transactions are not available for this block, this field will be null. + transactionCount: Int + # StateRoot is the keccak256 hash of the state trie after this block was processed. + stateRoot: Bytes32! + # ReceiptsRoot is the keccak256 hash of the trie of transaction receipts in this block. + receiptsRoot: Bytes32! + # Miner is the account that mined this block. + miner(block: Long): Account! + # ExtraData is an arbitrary data field supplied by the miner. + extraData: Bytes! + # GasLimit is the maximum amount of gas that was available to transactions in this block. + gasLimit: Long! + # GasUsed is the amount of gas that was used executing transactions in this block. + gasUsed: Long! + # Timestamp is the unix timestamp at which this block was mined. + timestamp: BigInt! + # LogsBloom is a bloom filter that can be used to check if a block may + # contain log entries matching a filter. + logsBloom: Bytes! + # MixHash is the hash that was used as an input to the PoW process. + mixHash: Bytes32! + # Difficulty is a measure of the difficulty of mining this block. + difficulty: BigInt! + # TotalDifficulty is the sum of all difficulty values up to and including + # this block. + totalDifficulty: BigInt! + # OmmerCount is the number of ommers (AKA uncles) associated with this + # block. If ommers are unavailable, this field will be null. + ommerCount: Int + # Ommers is a list of ommer (AKA uncle) blocks associated with this block. + # If ommers are unavailable, this field will be null. Depending on your + # node, the transactions, transactionAt, transactionCount, ommers, + # ommerCount and ommerAt fields may not be available on any ommer blocks. + ommers: [Block] + # OmmerAt returns the ommer (AKA uncle) at the specified index. If ommers + # are unavailable, or the index is out of bounds, this field will be null. + ommerAt(index: Int!): Block + # OmmerHash is the keccak256 hash of all the ommers (AKA uncles) + # associated with this block. + ommerHash: Bytes32! + # Transactions is a list of transactions associated with this block. If + # transactions are unavailable for this block, this field will be null. + transactions: [Transaction!] + # TransactionAt returns the transaction at the specified index. If + # transactions are unavailable for this block, or if the index is out of + # bounds, this field will be null. + transactionAt(index: Int!): Transaction + # Logs returns a filtered set of logs from this block. + logs(filter: BlockFilterCriteria!): [Log!]! + } + + # CallData represents the data associated with a local contract call. + # All fields are optional. + input CallData { + # From is the address making the call. + from: Address + # To is the address the call is sent to. + to: Address + # Gas is the amount of gas sent with the call. + gas: Long + # GasPrice is the price, in wei, offered for each unit of gas. + gasPrice: BigInt + # Value is the value, in wei, sent along with the call. + value: BigInt + # Data is the data sent to the callee. + data: Bytes + } + + # CallResult is the result of a local call operationn. + type CallResult { + # Data is the return data of the called contract. + data: Bytes! + # GasUsed is the amount of gas used by the call, after any refunds. + gasUsed: Long! + # Status is the result of the call - 1 for success or 0 for failure. + status: Long! + } + + # FilterCriteria encapsulates log filter criteria for searching log entries. + input FilterCriteria { + # FromBlock is the block at which to start searching, inclusive. Defaults + # to the latest block if not supplied. + fromBlock: Long + # ToBlock is the block at which to stop searching, inclusive. Defaults + # to the latest block if not supplied. + toBlock: Long + # Addresses is a list of addresses that are of interest. If this list is + # empty, results will not be filtered by address. + addresses: [Address!] + # Topics list restricts matches to particular event topics. Each event has a list + # of topics. Topics matches a prefix of that list. An empty element array matches any + # topic. Non-empty elements represent an alternative that matches any of the + # contained topics. + # + # Examples: + # - [] or nil matches any topic list + # - [[A]] matches topic A in first position + # - [[], [B]] matches any topic in first position, B in second position + # - [[A], [B]] matches topic A in first position, B in second position + # - [[A, B]], [C, D]] matches topic (A OR B) in first position, (C OR D) in second position + topics: [[Bytes32!]!] + } + + # SyncState contains the current synchronisation state of the client. + type SyncState{ + # StartingBlock is the block number at which synchronisation started. + startingBlock: Long! + # CurrentBlock is the point at which synchronisation has presently reached. + currentBlock: Long! + # HighestBlock is the latest known block number. + highestBlock: Long! + # PulledStates is the number of state entries fetched so far, or null + # if this is not known or not relevant. + pulledStates: Long + # KnownStates is the number of states the node knows of so far, or null + # if this is not known or not relevant. + knownStates: Long + } + + type Query { + # Account fetches an Ethereum account at the specified block number. + # If blockNumber is not provided, it defaults to the most recent block. + account(address: Address!, blockNumber: Long): Account! + # Block fetches an Ethereum block by number or by hash. If neither is + # supplied, the most recent known block is returned. + block(number: Long, hash: Bytes32): Block + # Blocks returns all the blocks between two numbers, inclusive. If + # to is not supplied, it defaults to the most recent known block. + blocks(from: Long!, to: Long): [Block!]! + # Transaction returns a transaction specified by its hash. + transaction(hash: Bytes32!): Transaction + # Call executes a local call operation. If blockNumber is not specified, + # it defaults to the most recent known block. + call(data: CallData!, blockNumber: Long): CallResult + # EstimateGas estimates the amount of gas that will be required for + # successful execution of a transaction. If blockNumber is not specified, + # it defaults ot the most recent known block. + estimateGas(data: CallData!, blockNumber: Long): Long! + # Logs returns log entries matching the provided filter. + logs(filter: FilterCriteria!): [Log!]! + # GasPrice returns the node's estimate of a gas price sufficient to + # ensure a transaction is mined in a timely fashion. + gasPrice: BigInt! + # ProtocolVersion returns the current wire protocol version number. + protocolVersion: Int! + # Syncing returns information on the current synchronisation state. + syncing: SyncState + } + + type Mutation { + # SendRawTransaction sends an RLP-encoded transaction to the network. + sendRawTransaction(data: Bytes!): Bytes32! + } +` diff --git a/interfaces.go b/interfaces.go index be78344062..1ff31f96b6 100644 --- a/interfaces.go +++ b/interfaces.go @@ -144,9 +144,9 @@ type FilterQuery struct { // Examples: // {} or nil matches any topic list // {{A}} matches topic A in first position - // {{}, {B}} matches any topic in first position, B in second position - // {{A}, {B}} matches topic A in first position, B in second position - // {{A, B}, {C, D}} matches topic (A OR B) in first position, (C OR D) in second position + // {{}, {B}} matches any topic in first position AND B in second position + // {{A}, {B}} matches topic A in first position AND B in second position + // {{A, B}, {C, D}} matches topic (A OR B) in first position AND (C OR D) in second position Topics [][]common.Hash } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 73b629bd9c..26dc1e8a01 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -378,7 +378,7 @@ func (s *PrivateAccountAPI) SendTransaction(ctx context.Context, args SendTxArgs log.Warn("Failed transaction send attempt", "from", args.From, "to", args.To, "value", args.Value.ToInt(), "err", err) return common.Hash{}, err } - return submitTransaction(ctx, s.b, signed) + return SubmitTransaction(ctx, s.b, signed) } // SignTransaction will create a transaction from the given arguments and @@ -675,41 +675,54 @@ func (s *PublicBlockChainAPI) GetStorageAt(ctx context.Context, address common.A // CallArgs represents the arguments for a call. type CallArgs struct { - From common.Address `json:"from"` + From *common.Address `json:"from"` To *common.Address `json:"to"` - Gas hexutil.Uint64 `json:"gas"` - GasPrice hexutil.Big `json:"gasPrice"` - Value hexutil.Big `json:"value"` - Data hexutil.Bytes `json:"data"` + Gas *hexutil.Uint64 `json:"gas"` + GasPrice *hexutil.Big `json:"gasPrice"` + Value *hexutil.Big `json:"value"` + Data *hexutil.Bytes `json:"data"` } -func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber, timeout time.Duration) ([]byte, uint64, bool, error) { +func DoCall(ctx context.Context, b Backend, args CallArgs, blockNr rpc.BlockNumber, vmCfg vm.Config, timeout time.Duration) ([]byte, uint64, bool, error) { defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) - state, header, err := s.b.StateAndHeaderByNumber(ctx, blockNr) + state, header, err := b.StateAndHeaderByNumber(ctx, blockNr) if state == nil || err != nil { return nil, 0, false, err } // Set sender address or use a default if none specified - addr := args.From - if addr == (common.Address{}) { - if wallets := s.b.AccountManager().Wallets(); len(wallets) > 0 { + var addr common.Address + if args.From == nil { + if wallets := b.AccountManager().Wallets(); len(wallets) > 0 { if accounts := wallets[0].Accounts(); len(accounts) > 0 { addr = accounts[0].Address } } + } else { + addr = *args.From } // Set default gas & gas price if none were set - gas, gasPrice := uint64(args.Gas), args.GasPrice.ToInt() - if gas == 0 { - gas = math.MaxUint64 / 2 + gas := uint64(math.MaxUint64 / 2) + if args.Gas != nil { + gas = uint64(*args.Gas) } - if gasPrice.Sign() == 0 { - gasPrice = new(big.Int).SetUint64(defaultGasPrice) + gasPrice := new(big.Int).SetUint64(defaultGasPrice) + if args.GasPrice != nil { + gasPrice = args.GasPrice.ToInt() + } + + value := new(big.Int) + if args.Value != nil { + value = args.Value.ToInt() + } + + var data []byte + if args.Data != nil { + data = []byte(*args.Data) } // Create new call message - msg := types.NewMessage(addr, args.To, 0, args.Value.ToInt(), gas, gasPrice, args.Data, false) + msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, data, false) // Setup context so it may be cancelled the call has completed // or, in case of unmetered gas, setup a context with a timeout. @@ -724,7 +737,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr defer cancel() // Get a new instance of the EVM. - evm, vmError, err := s.b.GetEVM(ctx, msg, state, header) + evm, vmError, err := b.GetEVM(ctx, msg, state, header) if err != nil { return nil, 0, false, err } @@ -748,24 +761,22 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr // Call executes the given transaction on the state for the given block number. // It doesn't make and changes in the state/blockchain and is useful to execute and retrieve values. func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { - result, _, _, err := s.doCall(ctx, args, blockNr, 5*time.Second) + result, _, _, err := DoCall(ctx, s.b, args, blockNr, vm.Config{}, 5*time.Second) return (hexutil.Bytes)(result), err } -// EstimateGas returns an estimate of the amount of gas needed to execute the -// given transaction against the current pending block. -func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (hexutil.Uint64, error) { +func DoEstimateGas(ctx context.Context, b Backend, args CallArgs, blockNr rpc.BlockNumber) (hexutil.Uint64, error) { // Binary search the gas requirement, as it may be higher than the amount used var ( lo uint64 = params.TxGas - 1 hi uint64 cap uint64 ) - if uint64(args.Gas) >= params.TxGas { - hi = uint64(args.Gas) + if args.Gas != nil && uint64(*args.Gas) >= params.TxGas { + hi = uint64(*args.Gas) } else { - // Retrieve the current pending block to act as the gas ceiling - block, err := s.b.BlockByNumber(ctx, rpc.PendingBlockNumber) + // Retrieve the block to act as the gas ceiling + block, err := b.BlockByNumber(ctx, blockNr) if err != nil { return 0, err } @@ -775,9 +786,9 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (h // Create a helper to check if a gas allowance results in an executable transaction executable := func(gas uint64) bool { - args.Gas = hexutil.Uint64(gas) + args.Gas = (*hexutil.Uint64)(&gas) - _, _, failed, err := s.doCall(ctx, args, rpc.PendingBlockNumber, 0) + _, _, failed, err := DoCall(ctx, b, args, rpc.PendingBlockNumber, vm.Config{}, 0) if err != nil || failed { return false } @@ -801,6 +812,12 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (h return hexutil.Uint64(hi), nil } +// EstimateGas returns an estimate of the amount of gas needed to execute the +// given transaction against the current pending block. +func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (hexutil.Uint64, error) { + return DoEstimateGas(ctx, s.b, args, rpc.PendingBlockNumber) +} + // ExecutionResult groups all structured logs emitted by the EVM // while replaying a transaction in debug mode as well as transaction // execution status, the amount of gas used and the return value @@ -825,7 +842,7 @@ type StructLogRes struct { Storage *map[string]string `json:"storage,omitempty"` } -// formatLogs formats EVM returned structured logs for json output +// FormatLogs formats EVM returned structured logs for json output func FormatLogs(logs []vm.StructLog) []StructLogRes { formatted := make([]StructLogRes, len(logs)) for index, trace := range logs { @@ -1256,8 +1273,8 @@ func (args *SendTxArgs) toTransaction() *types.Transaction { return types.NewTransaction(uint64(*args.Nonce), *args.To, (*big.Int)(args.Value), uint64(*args.Gas), (*big.Int)(args.GasPrice), input) } -// submitTransaction is a helper function that submits tx to txPool and logs a message. -func submitTransaction(ctx context.Context, b Backend, tx *types.Transaction) (common.Hash, error) { +// SubmitTransaction is a helper function that submits tx to txPool and logs a message. +func SubmitTransaction(ctx context.Context, b Backend, tx *types.Transaction) (common.Hash, error) { if err := b.SendTx(ctx, tx); err != nil { return common.Hash{}, err } @@ -1309,7 +1326,7 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Sen if err != nil { return common.Hash{}, err } - return submitTransaction(ctx, s.b, signed) + return SubmitTransaction(ctx, s.b, signed) } // SendRawTransaction will add the signed transaction to the transaction pool. @@ -1319,7 +1336,7 @@ func (s *PublicTransactionPoolAPI) SendRawTransaction(ctx context.Context, encod if err := rlp.DecodeBytes(encodedTx, tx); err != nil { return common.Hash{}, err } - return submitTransaction(ctx, s.b, tx) + return SubmitTransaction(ctx, s.b, tx) } // Sign calculates an ECDSA signature for: diff --git a/les/backend.go b/les/backend.go index d0db71019e..cd99f8f813 100644 --- a/les/backend.go +++ b/les/backend.go @@ -109,8 +109,12 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { bloomIndexer: eth.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), } + var trustedNodes []string + if leth.config.ULC != nil { + trustedNodes = leth.config.ULC.TrustedServers + } leth.relay = NewLesTxRelay(peers, leth.reqDist) - leth.serverPool = newServerPool(chainDb, quitSync, &leth.wg) + leth.serverPool = newServerPool(chainDb, quitSync, &leth.wg, trustedNodes) leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool) leth.odr = NewLesOdr(chainDb, light.DefaultClientIndexerConfig, leth.retriever) @@ -136,10 +140,33 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { } leth.txPool = light.NewTxPool(leth.chainConfig, leth.blockchain, leth.relay) - if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, light.DefaultClientIndexerConfig, true, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, leth.serverPool, quitSync, &leth.wg); err != nil { + + if leth.protocolManager, err = NewProtocolManager( + leth.chainConfig, + light.DefaultClientIndexerConfig, + true, + config.NetworkId, + leth.eventMux, + leth.engine, + leth.peers, + leth.blockchain, + nil, + chainDb, + leth.odr, + leth.relay, + leth.serverPool, + quitSync, + &leth.wg, + config.ULC); err != nil { return nil, err } + + if leth.protocolManager.isULCEnabled() { + log.Warn("Ultra light client is enabled", "trustedNodes", len(leth.protocolManager.ulc.trustedKeys), "minTrustedFraction", leth.protocolManager.ulc.minTrustedFraction) + leth.blockchain.DisableCheckFreq() + } leth.ApiBackend = &LesApiBackend{leth, nil} + gpoParams := config.GPO if gpoParams.Default == nil { gpoParams.Default = config.MinerGasPrice diff --git a/les/fetcher.go b/les/fetcher.go index 2615f69df9..aa3101af70 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -43,7 +43,7 @@ const ( type lightFetcher struct { pm *ProtocolManager odr *LesOdr - chain *light.LightChain + chain lightChain lock sync.Mutex // lock protects access to the fetcher's internal state variables except sent requests maxConfirmedTd *big.Int @@ -52,11 +52,19 @@ type lightFetcher struct { syncing bool syncDone chan *peer - reqMu sync.RWMutex // reqMu protects access to sent header fetch requests - requested map[uint64]fetchRequest - deliverChn chan fetchResponse - timeoutChn chan uint64 - requestChn chan bool // true if initiated from outside + reqMu sync.RWMutex // reqMu protects access to sent header fetch requests + requested map[uint64]fetchRequest + deliverChn chan fetchResponse + timeoutChn chan uint64 + requestChn chan bool // true if initiated from outside + lastTrustedHeader *types.Header +} + +// lightChain extends the BlockChain interface by locking. +type lightChain interface { + BlockChain + LockChain() + UnlockChain() } // fetcherPeerInfo holds fetcher-specific information about each active peer @@ -145,6 +153,7 @@ func (f *lightFetcher) syncLoop() { reqID uint64 syncing bool ) + if !f.syncing && !(newAnnounce && s) { rq, reqID, syncing = f.nextRequest() } @@ -227,7 +236,6 @@ func (f *lightFetcher) registerPeer(p *peer) { f.lock.Lock() defer f.lock.Unlock() - f.peers[p] = &fetcherPeerInfo{nodeByHash: make(map[common.Hash]*fetcherTreeNode)} } @@ -280,8 +288,10 @@ func (f *lightFetcher) announce(p *peer, head *announceData) { fp.nodeCnt = 0 fp.nodeByHash = make(map[common.Hash]*fetcherTreeNode) } + // check if the node count is too high to add new nodes, discard oldest ones if necessary if n != nil { - // check if the node count is too high to add new nodes, discard oldest ones if necessary + // n is now the reorg common ancestor, add a new branch of nodes + // check if the node count is too high to add new nodes locked := false for uint64(fp.nodeCnt)+head.Number-n.number > maxNodeCount && fp.root != nil { if !locked { @@ -325,6 +335,7 @@ func (f *lightFetcher) announce(p *peer, head *announceData) { fp.nodeByHash[n.hash] = n } } + if n == nil { // could not find reorg common ancestor or had to delete entire tree, a new root and a resync is needed if fp.root != nil { @@ -411,25 +422,13 @@ func (f *lightFetcher) requestedID(reqID uint64) bool { // to be downloaded starting from the head backwards is also returned func (f *lightFetcher) nextRequest() (*distReq, uint64, bool) { var ( - bestHash common.Hash - bestAmount uint64 + bestHash common.Hash + bestAmount uint64 + bestTd *big.Int + bestSyncing bool ) - bestTd := f.maxConfirmedTd - bestSyncing := false + bestHash, bestAmount, bestTd, bestSyncing = f.findBestRequest() - for p, fp := range f.peers { - for hash, n := range fp.nodeByHash { - if !f.checkKnownNode(p, n) && !n.requested && (bestTd == nil || n.td.Cmp(bestTd) >= 0) { - amount := f.requestAmount(p, n) - if bestTd == nil || n.td.Cmp(bestTd) > 0 || amount < bestAmount { - bestHash = hash - bestAmount = amount - bestTd = n.td - bestSyncing = fp.bestConfirmed == nil || fp.root == nil || !f.checkKnownNode(p, fp.root) - } - } - } - } if bestTd == f.maxConfirmedTd { return nil, 0, false } @@ -437,74 +436,142 @@ func (f *lightFetcher) nextRequest() (*distReq, uint64, bool) { var rq *distReq reqID := genReqID() if bestSyncing { - rq = &distReq{ - getCost: func(dp distPeer) uint64 { - return 0 - }, - canSend: func(dp distPeer) bool { - p := dp.(*peer) - f.lock.Lock() - defer f.lock.Unlock() - - fp := f.peers[p] - return fp != nil && fp.nodeByHash[bestHash] != nil - }, - request: func(dp distPeer) func() { - go func() { - p := dp.(*peer) - p.Log().Debug("Synchronisation started") - f.pm.synchronise(p) - f.syncDone <- p - }() - return nil - }, - } + rq = f.newFetcherDistReqForSync(bestHash) } else { - rq = &distReq{ - getCost: func(dp distPeer) uint64 { - p := dp.(*peer) - return p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount)) - }, - canSend: func(dp distPeer) bool { - p := dp.(*peer) - f.lock.Lock() - defer f.lock.Unlock() - - fp := f.peers[p] - if fp == nil { - return false - } - n := fp.nodeByHash[bestHash] - return n != nil && !n.requested - }, - request: func(dp distPeer) func() { - p := dp.(*peer) - f.lock.Lock() - fp := f.peers[p] - if fp != nil { - n := fp.nodeByHash[bestHash] - if n != nil { - n.requested = true - } - } - f.lock.Unlock() - - cost := p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount)) - p.fcServer.QueueRequest(reqID, cost) - f.reqMu.Lock() - f.requested[reqID] = fetchRequest{hash: bestHash, amount: bestAmount, peer: p, sent: mclock.Now()} - f.reqMu.Unlock() - go func() { - time.Sleep(hardRequestTimeout) - f.timeoutChn <- reqID - }() - return func() { p.RequestHeadersByHash(reqID, cost, bestHash, int(bestAmount), 0, true) } - }, - } + rq = f.newFetcherDistReq(bestHash, reqID, bestAmount) } return rq, reqID, bestSyncing } +// findBestRequest finds the best head to request that has been announced by but not yet requested from a known peer. +// It also returns the announced Td (which should be verified after fetching the head), +// the necessary amount to request and whether a downloader sync is necessary instead of a normal header request. +func (f *lightFetcher) findBestRequest() (bestHash common.Hash, bestAmount uint64, bestTd *big.Int, bestSyncing bool) { + bestTd = f.maxConfirmedTd + bestSyncing = false + + for p, fp := range f.peers { + for hash, n := range fp.nodeByHash { + if f.checkKnownNode(p, n) || n.requested { + continue + } + + //if ulc mode is disabled, isTrustedHash returns true + amount := f.requestAmount(p, n) + if (bestTd == nil || n.td.Cmp(bestTd) > 0 || amount < bestAmount) && (f.isTrustedHash(hash) || f.maxConfirmedTd.Int64() == 0) { + bestHash = hash + bestTd = n.td + bestAmount = amount + bestSyncing = fp.bestConfirmed == nil || fp.root == nil || !f.checkKnownNode(p, fp.root) + } + } + } + return +} + +// isTrustedHash checks if the block can be trusted by the minimum trusted fraction. +func (f *lightFetcher) isTrustedHash(hash common.Hash) bool { + if !f.pm.isULCEnabled() { + return true + } + + var numAgreed int + for p, fp := range f.peers { + if !p.isTrusted { + continue + } + if _, ok := fp.nodeByHash[hash]; !ok { + continue + } + + numAgreed++ + } + + return 100*numAgreed/len(f.pm.ulc.trustedKeys) >= f.pm.ulc.minTrustedFraction +} + +func (f *lightFetcher) newFetcherDistReqForSync(bestHash common.Hash) *distReq { + return &distReq{ + getCost: func(dp distPeer) uint64 { + return 0 + }, + canSend: func(dp distPeer) bool { + p := dp.(*peer) + f.lock.Lock() + defer f.lock.Unlock() + + if p.isOnlyAnnounce { + return false + } + + fp := f.peers[p] + return fp != nil && fp.nodeByHash[bestHash] != nil + }, + request: func(dp distPeer) func() { + if f.pm.isULCEnabled() { + //keep last trusted header before sync + f.setLastTrustedHeader(f.chain.CurrentHeader()) + } + go func() { + p := dp.(*peer) + p.Log().Debug("Synchronisation started") + f.pm.synchronise(p) + f.syncDone <- p + }() + return nil + }, + } +} + +// newFetcherDistReq creates a new request for the distributor. +func (f *lightFetcher) newFetcherDistReq(bestHash common.Hash, reqID uint64, bestAmount uint64) *distReq { + return &distReq{ + getCost: func(dp distPeer) uint64 { + p := dp.(*peer) + return p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount)) + }, + canSend: func(dp distPeer) bool { + p := dp.(*peer) + f.lock.Lock() + defer f.lock.Unlock() + + if p.isOnlyAnnounce { + return false + } + + fp := f.peers[p] + if fp == nil { + return false + } + n := fp.nodeByHash[bestHash] + return n != nil && !n.requested + }, + request: func(dp distPeer) func() { + p := dp.(*peer) + f.lock.Lock() + fp := f.peers[p] + if fp != nil { + n := fp.nodeByHash[bestHash] + if n != nil { + n.requested = true + } + } + f.lock.Unlock() + + cost := p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount)) + p.fcServer.QueueRequest(reqID, cost) + f.reqMu.Lock() + f.requested[reqID] = fetchRequest{hash: bestHash, amount: bestAmount, peer: p, sent: mclock.Now()} + f.reqMu.Unlock() + go func() { + time.Sleep(hardRequestTimeout) + f.timeoutChn <- reqID + }() + return func() { p.RequestHeadersByHash(reqID, cost, bestHash, int(bestAmount), 0, true) } + }, + } +} + // deliverHeaders delivers header download request responses for processing func (f *lightFetcher) deliverHeaders(peer *peer, reqID uint64, headers []*types.Header) { f.deliverChn <- fetchResponse{reqID: reqID, headers: headers, peer: peer} @@ -520,6 +587,7 @@ func (f *lightFetcher) processResponse(req fetchRequest, resp fetchResponse) boo for i, header := range resp.headers { headers[int(req.amount)-1-i] = header } + if _, err := f.chain.InsertHeaderChain(headers, 1); err != nil { if err == consensus.ErrFutureBlock { return true @@ -544,6 +612,7 @@ func (f *lightFetcher) processResponse(req fetchRequest, resp fetchResponse) boo // downloaded and validated batch or headers func (f *lightFetcher) newHeaders(headers []*types.Header, tds []*big.Int) { var maxTd *big.Int + for p, fp := range f.peers { if !f.checkAnnouncedHeaders(fp, headers, tds) { p.Log().Debug("Inconsistent announcement") @@ -553,6 +622,7 @@ func (f *lightFetcher) newHeaders(headers []*types.Header, tds []*big.Int) { maxTd = fp.confirmedTd } } + if maxTd != nil { f.updateMaxConfirmedTd(maxTd) } @@ -640,22 +710,72 @@ func (f *lightFetcher) checkSyncedHeaders(p *peer) { p.Log().Debug("Unknown peer to check sync headers") return } + n := fp.lastAnnounced var td *big.Int + + var h *types.Header + if f.pm.isULCEnabled() { + var unapprovedHashes []common.Hash + // Overwrite last announced for ULC mode + h, unapprovedHashes = f.lastTrustedTreeNode(p) + //rollback untrusted blocks + f.chain.Rollback(unapprovedHashes) + //overwrite to last trusted + n = fp.nodeByHash[h.Hash()] + } + + //find last valid block for n != nil { if td = f.chain.GetTd(n.hash, n.number); td != nil { break } n = n.parent } - // now n is the latest downloaded header after syncing + + // Now n is the latest downloaded/approved header after syncing if n == nil { p.Log().Debug("Synchronisation failed") go f.pm.removePeer(p.id) - } else { - header := f.chain.GetHeader(n.hash, n.number) - f.newHeaders([]*types.Header{header}, []*big.Int{td}) + return } + header := f.chain.GetHeader(n.hash, n.number) + f.newHeaders([]*types.Header{header}, []*big.Int{td}) +} + +// lastTrustedTreeNode return last approved treeNode and a list of unapproved hashes +func (f *lightFetcher) lastTrustedTreeNode(p *peer) (*types.Header, []common.Hash) { + unapprovedHashes := make([]common.Hash, 0) + current := f.chain.CurrentHeader() + + if f.lastTrustedHeader == nil { + return current, unapprovedHashes + } + + canonical := f.chain.CurrentHeader() + if canonical.Number.Uint64() > f.lastTrustedHeader.Number.Uint64() { + canonical = f.chain.GetHeaderByNumber(f.lastTrustedHeader.Number.Uint64()) + } + commonAncestor := rawdb.FindCommonAncestor(f.pm.chainDb, canonical, f.lastTrustedHeader) + if commonAncestor == nil { + log.Error("Common ancestor of last trusted header and canonical header is nil", "canonical hash", canonical.Hash(), "trusted hash", f.lastTrustedHeader.Hash()) + return current, unapprovedHashes + } + + for current.Hash() == commonAncestor.Hash() { + if f.isTrustedHash(current.Hash()) { + break + } + unapprovedHashes = append(unapprovedHashes, current.Hash()) + current = f.chain.GetHeader(current.ParentHash, current.Number.Uint64()-1) + } + return current, unapprovedHashes +} + +func (f *lightFetcher) setLastTrustedHeader(h *types.Header) { + f.lock.Lock() + defer f.lock.Unlock() + f.lastTrustedHeader = h } // checkKnownNode checks if a block tree node is known (downloaded and validated) @@ -747,6 +867,7 @@ func (f *lightFetcher) updateMaxConfirmedTd(td *big.Int) { if f.lastUpdateStats != nil { f.lastUpdateStats.next = newEntry } + f.lastUpdateStats = newEntry for p := range f.peers { f.checkUpdateStats(p, newEntry) @@ -769,6 +890,7 @@ func (f *lightFetcher) checkUpdateStats(p *peer, newEntry *updateStatsEntry) { p.Log().Debug("Unknown peer to check update stats") return } + if newEntry != nil && fp.firstUpdateStats == nil { fp.firstUpdateStats = newEntry } diff --git a/les/fetcher_test.go b/les/fetcher_test.go new file mode 100644 index 0000000000..4ae7f6d04a --- /dev/null +++ b/les/fetcher_test.go @@ -0,0 +1,155 @@ +package les + +import ( + "math/big" + "testing" + + "net" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +func TestFetcherULCPeerSelector(t *testing.T) { + + var ( + id1 enode.ID = newNodeID(t).ID() + id2 enode.ID = newNodeID(t).ID() + id3 enode.ID = newNodeID(t).ID() + id4 enode.ID = newNodeID(t).ID() + ) + + ftn1 := &fetcherTreeNode{ + hash: common.HexToHash("1"), + td: big.NewInt(1), + } + ftn2 := &fetcherTreeNode{ + hash: common.HexToHash("2"), + td: big.NewInt(2), + parent: ftn1, + } + ftn3 := &fetcherTreeNode{ + hash: common.HexToHash("3"), + td: big.NewInt(3), + parent: ftn2, + } + lf := lightFetcher{ + pm: &ProtocolManager{ + ulc: &ulc{ + trustedKeys: map[string]struct{}{ + id1.String(): {}, + id2.String(): {}, + id3.String(): {}, + id4.String(): {}, + }, + minTrustedFraction: 70, + }, + }, + maxConfirmedTd: ftn1.td, + + peers: map[*peer]*fetcherPeerInfo{ + { + id: "peer1", + Peer: p2p.NewPeer(id1, "peer1", []p2p.Cap{}), + isTrusted: true, + }: { + nodeByHash: map[common.Hash]*fetcherTreeNode{ + ftn1.hash: ftn1, + ftn2.hash: ftn2, + }, + }, + { + Peer: p2p.NewPeer(id2, "peer2", []p2p.Cap{}), + id: "peer2", + isTrusted: true, + }: { + nodeByHash: map[common.Hash]*fetcherTreeNode{ + ftn1.hash: ftn1, + ftn2.hash: ftn2, + }, + }, + { + id: "peer3", + Peer: p2p.NewPeer(id3, "peer3", []p2p.Cap{}), + isTrusted: true, + }: { + nodeByHash: map[common.Hash]*fetcherTreeNode{ + ftn1.hash: ftn1, + ftn2.hash: ftn2, + ftn3.hash: ftn3, + }, + }, + { + id: "peer4", + Peer: p2p.NewPeer(id4, "peer4", []p2p.Cap{}), + isTrusted: true, + }: { + nodeByHash: map[common.Hash]*fetcherTreeNode{ + ftn1.hash: ftn1, + }, + }, + }, + chain: &lightChainStub{ + tds: map[common.Hash]*big.Int{}, + headers: map[common.Hash]*types.Header{ + ftn1.hash: {}, + ftn2.hash: {}, + ftn3.hash: {}, + }, + }, + } + bestHash, bestAmount, bestTD, sync := lf.findBestRequest() + + if bestTD == nil { + t.Fatal("Empty result") + } + + if bestTD.Cmp(ftn2.td) != 0 { + t.Fatal("bad td", bestTD) + } + if bestHash != ftn2.hash { + t.Fatal("bad hash", bestTD) + } + + _, _ = bestAmount, sync +} + +type lightChainStub struct { + BlockChain + tds map[common.Hash]*big.Int + headers map[common.Hash]*types.Header + insertHeaderChainAssertFunc func(chain []*types.Header, checkFreq int) (int, error) +} + +func (l *lightChainStub) GetHeader(hash common.Hash, number uint64) *types.Header { + if h, ok := l.headers[hash]; ok { + return h + } + + return nil +} + +func (l *lightChainStub) LockChain() {} +func (l *lightChainStub) UnlockChain() {} + +func (l *lightChainStub) GetTd(hash common.Hash, number uint64) *big.Int { + if td, ok := l.tds[hash]; ok { + return td + } + return nil +} + +func (l *lightChainStub) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) { + return l.insertHeaderChainAssertFunc(chain, checkFreq) +} + +func newNodeID(t *testing.T) *enode.Node { + key, err := crypto.GenerateKey() + if err != nil { + t.Fatal("generate key err:", err) + } + return enode.NewV4(&key.PublicKey, net.IP{}, 35000, 35000) +} diff --git a/les/handler.go b/les/handler.go index 19ccbcd2b6..46a1ed2d7b 100644 --- a/les/handler.go +++ b/les/handler.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -119,12 +120,29 @@ type ProtocolManager struct { // wait group is used for graceful shutdowns during downloading // and processing - wg *sync.WaitGroup + wg *sync.WaitGroup + ulc *ulc } // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(chainConfig *params.ChainConfig, indexerConfig *light.IndexerConfig, lightSync bool, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, serverPool *serverPool, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { +func NewProtocolManager( + chainConfig *params.ChainConfig, + indexerConfig *light.IndexerConfig, + lightSync bool, + networkId uint64, + mux *event.TypeMux, + engine consensus.Engine, + peers *peerSet, + blockchain BlockChain, + txpool txPool, + chainDb ethdb.Database, + odr *LesOdr, + txrelay *LesTxRelay, + serverPool *serverPool, + quitSync chan struct{}, + wg *sync.WaitGroup, + ulcConfig *eth.ULCConfig) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ lightSync: lightSync, @@ -149,6 +167,10 @@ func NewProtocolManager(chainConfig *params.ChainConfig, indexerConfig *light.In manager.reqDist = odr.retriever.dist } + if ulcConfig != nil { + manager.ulc = newULC(ulcConfig) + } + removePeer := manager.removePeer if disableClientRemovePeer { removePeer = func(id string) {} @@ -234,7 +256,11 @@ func (pm *ProtocolManager) runPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWrit } func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { - return newPeer(pv, nv, p, newMeteredMsgWriter(rw)) + var isTrusted bool + if pm.isULCEnabled() { + isTrusted = pm.ulc.isTrusted(p.ID()) + } + return newPeer(pv, nv, isTrusted, p, newMeteredMsgWriter(rw)) } // handle is the callback invoked to manage the life cycle of a les peer. When @@ -276,6 +302,7 @@ func (pm *ProtocolManager) handle(p *peer) error { if rw, ok := p.rw.(*meteredMsgReadWriter); ok { rw.Init(p.version) } + // Register the peer locally if err := pm.peers.Register(p); err != nil { p.Log().Error("Light Ethereum peer registration failed", "err", err) @@ -287,6 +314,7 @@ func (pm *ProtocolManager) handle(p *peer) error { } pm.removePeer(p.id) }() + // Register the peer in the downloader. If the downloader considers it banned, we disconnect if pm.lightSync { p.lock.Lock() @@ -371,16 +399,15 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Block header query, collect the requested headers and reply case AnnounceMsg: p.Log().Trace("Received announce message") - if p.requestAnnounceType == announceTypeNone { + if p.announceType == announceTypeNone { return errResp(ErrUnexpectedResponse, "") } - var req announceData if err := msg.Decode(&req); err != nil { return errResp(ErrDecode, "%v: %v", msg, err) } - if p.requestAnnounceType == announceTypeSigned { + if p.announceType == announceTypeSigned { if err := req.checkSignature(p.ID()); err != nil { p.Log().Trace("Invalid announcement signature", "err", err) return err @@ -1175,6 +1202,14 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { return stats } +// isULCEnabled returns true if we can use ULC +func (pm *ProtocolManager) isULCEnabled() bool { + if pm.ulc == nil || len(pm.ulc.trustedKeys) == 0 { + return false + } + return true +} + // downloaderPeerNotify implements peerSetNotify type downloaderPeerNotify ProtocolManager diff --git a/les/handler_test.go b/les/handler_test.go index 43be7f41b1..72ba266b3a 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -494,7 +494,7 @@ func TestGetBloombitsProofs(t *testing.T) { func TestTransactionStatusLes2(t *testing.T) { db := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, 0, nil, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, 0, nil, nil, nil, db, nil) chain := pm.blockchain.(*core.BlockChain) config := core.DefaultTxPoolConfig config.Journal = "" diff --git a/les/helper_test.go b/les/helper_test.go index b46d41f174..02b1668c84 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -146,7 +146,7 @@ func testRCL() RequestCostList { // newTestProtocolManager creates a new protocol manager for testing purposes, // with the given number of blocks already known, potential notification // channels for different events and relative chain indexers array. -func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *core.BlockGen), odr *LesOdr, peers *peerSet, db ethdb.Database) (*ProtocolManager, error) { +func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *core.BlockGen), odr *LesOdr, peers *peerSet, db ethdb.Database, ulcConfig *eth.ULCConfig) (*ProtocolManager, error) { var ( evmux = new(event.TypeMux) engine = ethash.NewFaker() @@ -176,7 +176,7 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor if lightSync { indexConfig = light.TestClientIndexerConfig } - pm, err := NewProtocolManager(gspec.Config, indexConfig, lightSync, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup)) + pm, err := NewProtocolManager(gspec.Config, indexConfig, lightSync, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup), ulcConfig) if err != nil { return nil, err } @@ -200,8 +200,8 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor // with the given number of blocks already known, potential notification // channels for different events and relative chain indexers array. In case of an error, the constructor force- // fails the test. -func newTestProtocolManagerMust(t *testing.T, lightSync bool, blocks int, generator func(int, *core.BlockGen), odr *LesOdr, peers *peerSet, db ethdb.Database) *ProtocolManager { - pm, err := newTestProtocolManager(lightSync, blocks, generator, odr, peers, db) +func newTestProtocolManagerMust(t *testing.T, lightSync bool, blocks int, generator func(int, *core.BlockGen), odr *LesOdr, peers *peerSet, db ethdb.Database, ulcConfig *eth.ULCConfig) *ProtocolManager { + pm, err := newTestProtocolManager(lightSync, blocks, generator, odr, peers, db, ulcConfig) if err != nil { t.Fatalf("Failed to create protocol manager: %v", err) } @@ -343,7 +343,7 @@ func newServerEnv(t *testing.T, blocks int, protocol int, waitIndexers func(*cor db := ethdb.NewMemDatabase() cIndexer, bIndexer, btIndexer := testIndexers(db, nil, light.TestServerIndexerConfig) - pm := newTestProtocolManagerMust(t, false, blocks, testChainGen, nil, nil, db) + pm := newTestProtocolManagerMust(t, false, blocks, testChainGen, nil, nil, db, nil) peer, _ := newTestPeer(t, "peer", protocol, pm, true) cIndexer.Start(pm.blockchain.(*core.BlockChain)) @@ -383,8 +383,8 @@ func newClientServerEnv(t *testing.T, blocks int, protocol int, waitIndexers fun lcIndexer, lbIndexer, lbtIndexer := testIndexers(ldb, odr, light.TestClientIndexerConfig) odr.SetIndexers(lcIndexer, lbtIndexer, lbIndexer) - pm := newTestProtocolManagerMust(t, false, blocks, testChainGen, nil, peers, db) - lpm := newTestProtocolManagerMust(t, true, 0, nil, odr, lPeers, ldb) + pm := newTestProtocolManagerMust(t, false, blocks, testChainGen, nil, peers, db, nil) + lpm := newTestProtocolManagerMust(t, true, 0, nil, odr, lPeers, ldb, nil) startIndexers := func(clientMode bool, pm *ProtocolManager) { if clientMode { diff --git a/les/odr.go b/les/odr.go index 9def05a676..f7592354de 100644 --- a/les/odr.go +++ b/les/odr.go @@ -109,7 +109,10 @@ func (odr *LesOdr) Retrieve(ctx context.Context, req light.OdrRequest) (err erro }, canSend: func(dp distPeer) bool { p := dp.(*peer) - return lreq.CanSend(p) + if !p.isOnlyAnnounce { + return lreq.CanSend(p) + } + return false }, request: func(dp distPeer) func() { p := dp.(*peer) diff --git a/les/peer.go b/les/peer.go index 678384f0eb..9ae94b20fe 100644 --- a/les/peer.go +++ b/les/peer.go @@ -56,7 +56,7 @@ type peer struct { version int // Protocol version negotiated network uint64 // Network ID being on - announceType, requestAnnounceType uint64 + announceType uint64 id string @@ -74,9 +74,12 @@ type peer struct { fcServer *flowcontrol.ServerNode // nil if the peer is client only fcServerParams *flowcontrol.ServerParams fcCosts requestCostTable + + isTrusted bool + isOnlyAnnounce bool } -func newPeer(version int, network uint64, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { +func newPeer(version int, network uint64, isTrusted bool, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { id := p.ID() return &peer{ @@ -86,6 +89,7 @@ func newPeer(version int, network uint64, p *p2p.Peer, rw p2p.MsgReadWriter) *pe network: network, id: fmt.Sprintf("%x", id[:8]), announceChn: make(chan announceData, 20), + isTrusted: isTrusted, } } @@ -401,23 +405,32 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis send = send.add("headNum", headNum) send = send.add("genesisHash", genesis) if server != nil { - send = send.add("serveHeaders", nil) - send = send.add("serveChainSince", uint64(0)) - send = send.add("serveStateSince", uint64(0)) - send = send.add("txRelay", nil) + if !server.onlyAnnounce { + //only announce server. It sends only announse requests + send = send.add("serveHeaders", nil) + send = send.add("serveChainSince", uint64(0)) + send = send.add("serveStateSince", uint64(0)) + send = send.add("txRelay", nil) + } send = send.add("flowControl/BL", server.defParams.BufLimit) send = send.add("flowControl/MRR", server.defParams.MinRecharge) list := server.fcCostStats.getCurrentList() send = send.add("flowControl/MRC", list) p.fcCosts = list.decode() } else { - p.requestAnnounceType = announceTypeSimple // set to default until "very light" client mode is implemented - send = send.add("announceType", p.requestAnnounceType) + //on client node + p.announceType = announceTypeSimple + if p.isTrusted { + p.announceType = announceTypeSigned + } + send = send.add("announceType", p.announceType) } + recvList, err := p.sendReceiveHandshake(send) if err != nil { return err } + recv := recvList.decode() var rGenesis, rHash common.Hash @@ -452,25 +465,33 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis if int(rVersion) != p.version { return errResp(ErrProtocolVersionMismatch, "%d (!= %d)", rVersion, p.version) } + if server != nil { // until we have a proper peer connectivity API, allow LES connection to other servers /*if recv.get("serveStateSince", nil) == nil { return errResp(ErrUselessPeer, "wanted client, got server") }*/ if recv.get("announceType", &p.announceType) != nil { + //set default announceType on server side p.announceType = announceTypeSimple } p.fcClient = flowcontrol.NewClientNode(server.fcManager, server.defParams) } else { + //mark OnlyAnnounce server if "serveHeaders", "serveChainSince", "serveStateSince" or "txRelay" fields don't exist if recv.get("serveChainSince", nil) != nil { - return errResp(ErrUselessPeer, "peer cannot serve chain") + p.isOnlyAnnounce = true } if recv.get("serveStateSince", nil) != nil { - return errResp(ErrUselessPeer, "peer cannot serve state") + p.isOnlyAnnounce = true } if recv.get("txRelay", nil) != nil { - return errResp(ErrUselessPeer, "peer cannot relay transactions") + p.isOnlyAnnounce = true } + + if p.isOnlyAnnounce && !p.isTrusted { + return errResp(ErrUselessPeer, "peer cannot serve requests") + } + params := &flowcontrol.ServerParams{} if err := recv.get("flowControl/BL", ¶ms.BufLimit); err != nil { return err @@ -486,7 +507,6 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis p.fcServer = flowcontrol.NewServerNode(params) p.fcCosts = MRC.decode() } - p.headInfo = &announceData{Td: rTd, Hash: rHash, Number: rNum} return nil } @@ -576,8 +596,10 @@ func (ps *peerSet) Unregister(id string) error { for _, n := range peers { n.unregisterPeer(p) } + p.sendQueue.quit() p.Peer.Disconnect(p2p.DiscUselessPeer) + return nil } } diff --git a/les/peer_test.go b/les/peer_test.go new file mode 100644 index 0000000000..0ba9cbfd52 --- /dev/null +++ b/les/peer_test.go @@ -0,0 +1,297 @@ +package les + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/les/flowcontrol" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rlp" +) + +const ( + test_networkid = 10 + protocol_version = 2123 +) + +var ( + hash = common.HexToHash("some string") + genesis = common.HexToHash("genesis hash") + headNum = uint64(1234) + td = big.NewInt(123) +) + +//ulc connects to trusted peer and send announceType=announceTypeSigned +func TestPeerHandshakeSetAnnounceTypeToAnnounceTypeSignedForTrustedPeer(t *testing.T) { + + var id enode.ID = newNodeID(t).ID() + + //peer to connect(on ulc side) + p := peer{ + Peer: p2p.NewPeer(id, "test peer", []p2p.Cap{}), + version: protocol_version, + isTrusted: true, + rw: &rwStub{ + WriteHook: func(recvList keyValueList) { + //checking that ulc sends to peer allowedRequests=onlyAnnounceRequests and announceType = announceTypeSigned + recv := recvList.decode() + var reqType uint64 + + err := recv.get("announceType", &reqType) + if err != nil { + t.Fatal(err) + } + + if reqType != announceTypeSigned { + t.Fatal("Expected announceTypeSigned") + } + }, + ReadHook: func(l keyValueList) keyValueList { + l = l.add("serveHeaders", nil) + l = l.add("serveChainSince", uint64(0)) + l = l.add("serveStateSince", uint64(0)) + l = l.add("txRelay", nil) + l = l.add("flowControl/BL", uint64(0)) + l = l.add("flowControl/MRR", uint64(0)) + l = l.add("flowControl/MRC", RequestCostList{}) + + return l + }, + }, + network: test_networkid, + } + + err := p.Handshake(td, hash, headNum, genesis, nil) + if err != nil { + t.Fatalf("Handshake error: %s", err) + } + + if p.announceType != announceTypeSigned { + t.Fatal("Incorrect announceType") + } +} + +func TestPeerHandshakeAnnounceTypeSignedForTrustedPeersPeerNotInTrusted(t *testing.T) { + var id enode.ID = newNodeID(t).ID() + p := peer{ + Peer: p2p.NewPeer(id, "test peer", []p2p.Cap{}), + version: protocol_version, + rw: &rwStub{ + WriteHook: func(recvList keyValueList) { + //checking that ulc sends to peer allowedRequests=noRequests and announceType != announceTypeSigned + recv := recvList.decode() + var reqType uint64 + + err := recv.get("announceType", &reqType) + if err != nil { + t.Fatal(err) + } + + if reqType == announceTypeSigned { + t.Fatal("Expected not announceTypeSigned") + } + }, + ReadHook: func(l keyValueList) keyValueList { + l = l.add("serveHeaders", nil) + l = l.add("serveChainSince", uint64(0)) + l = l.add("serveStateSince", uint64(0)) + l = l.add("txRelay", nil) + l = l.add("flowControl/BL", uint64(0)) + l = l.add("flowControl/MRR", uint64(0)) + l = l.add("flowControl/MRC", RequestCostList{}) + + return l + }, + }, + network: test_networkid, + } + + err := p.Handshake(td, hash, headNum, genesis, nil) + if err != nil { + t.Fatal(err) + } + if p.announceType == announceTypeSigned { + t.Fatal("Incorrect announceType") + } +} + +func TestPeerHandshakeDefaultAllRequests(t *testing.T) { + var id enode.ID = newNodeID(t).ID() + + s := generateLesServer() + + p := peer{ + Peer: p2p.NewPeer(id, "test peer", []p2p.Cap{}), + version: protocol_version, + rw: &rwStub{ + ReadHook: func(l keyValueList) keyValueList { + l = l.add("announceType", uint64(announceTypeSigned)) + l = l.add("allowedRequests", uint64(0)) + + return l + }, + }, + network: test_networkid, + } + + err := p.Handshake(td, hash, headNum, genesis, s) + if err != nil { + t.Fatal(err) + } + + if p.isOnlyAnnounce { + t.Fatal("Incorrect announceType") + } +} + +func TestPeerHandshakeServerSendOnlyAnnounceRequestsHeaders(t *testing.T) { + var id enode.ID = newNodeID(t).ID() + + s := generateLesServer() + s.onlyAnnounce = true + + p := peer{ + Peer: p2p.NewPeer(id, "test peer", []p2p.Cap{}), + version: protocol_version, + rw: &rwStub{ + ReadHook: func(l keyValueList) keyValueList { + l = l.add("announceType", uint64(announceTypeSigned)) + + return l + }, + WriteHook: func(l keyValueList) { + for _, v := range l { + if v.Key == "serveHeaders" || + v.Key == "serveChainSince" || + v.Key == "serveStateSince" || + v.Key == "txRelay" { + t.Fatalf("%v exists", v.Key) + } + } + }, + }, + network: test_networkid, + } + + err := p.Handshake(td, hash, headNum, genesis, s) + if err != nil { + t.Fatal(err) + } +} +func TestPeerHandshakeClientReceiveOnlyAnnounceRequestsHeaders(t *testing.T) { + var id enode.ID = newNodeID(t).ID() + + p := peer{ + Peer: p2p.NewPeer(id, "test peer", []p2p.Cap{}), + version: protocol_version, + rw: &rwStub{ + ReadHook: func(l keyValueList) keyValueList { + l = l.add("flowControl/BL", uint64(0)) + l = l.add("flowControl/MRR", uint64(0)) + l = l.add("flowControl/MRC", RequestCostList{}) + + l = l.add("announceType", uint64(announceTypeSigned)) + + return l + }, + }, + network: test_networkid, + isTrusted: true, + } + + err := p.Handshake(td, hash, headNum, genesis, nil) + if err != nil { + t.Fatal(err) + } + + if !p.isOnlyAnnounce { + t.Fatal("isOnlyAnnounce must be true") + } +} + +func TestPeerHandshakeClientReturnErrorOnUselessPeer(t *testing.T) { + var id enode.ID = newNodeID(t).ID() + + p := peer{ + Peer: p2p.NewPeer(id, "test peer", []p2p.Cap{}), + version: protocol_version, + rw: &rwStub{ + ReadHook: func(l keyValueList) keyValueList { + l = l.add("flowControl/BL", uint64(0)) + l = l.add("flowControl/MRR", uint64(0)) + l = l.add("flowControl/MRC", RequestCostList{}) + + l = l.add("announceType", uint64(announceTypeSigned)) + + return l + }, + }, + network: test_networkid, + } + + err := p.Handshake(td, hash, headNum, genesis, nil) + if err == nil { + t.FailNow() + } +} + +func generateLesServer() *LesServer { + s := &LesServer{ + defParams: &flowcontrol.ServerParams{ + BufLimit: uint64(300000000), + MinRecharge: uint64(50000), + }, + fcManager: flowcontrol.NewClientManager(1, 2, 3), + fcCostStats: &requestCostStats{ + stats: make(map[uint64]*linReg, len(reqList)), + }, + } + for _, code := range reqList { + s.fcCostStats.stats[code] = &linReg{cnt: 100} + } + return s +} + +type rwStub struct { + ReadHook func(l keyValueList) keyValueList + WriteHook func(l keyValueList) +} + +func (s *rwStub) ReadMsg() (p2p.Msg, error) { + payload := keyValueList{} + payload = payload.add("protocolVersion", uint64(protocol_version)) + payload = payload.add("networkId", uint64(test_networkid)) + payload = payload.add("headTd", td) + payload = payload.add("headHash", hash) + payload = payload.add("headNum", headNum) + payload = payload.add("genesisHash", genesis) + + if s.ReadHook != nil { + payload = s.ReadHook(payload) + } + + size, p, err := rlp.EncodeToReader(payload) + if err != nil { + return p2p.Msg{}, err + } + + return p2p.Msg{ + Size: uint32(size), + Payload: p, + }, nil +} + +func (s *rwStub) WriteMsg(m p2p.Msg) error { + recvList := keyValueList{} + if err := m.Decode(&recvList); err != nil { + return err + } + + if s.WriteHook != nil { + s.WriteHook(recvList) + } + + return nil +} diff --git a/les/server.go b/les/server.go index 2fa0456d69..2ded3c184b 100644 --- a/les/server.go +++ b/les/server.go @@ -41,17 +41,34 @@ import ( type LesServer struct { lesCommons - fcManager *flowcontrol.ClientManager // nil if our node is client only - fcCostStats *requestCostStats - defParams *flowcontrol.ServerParams - lesTopics []discv5.Topic - privateKey *ecdsa.PrivateKey - quitSync chan struct{} + fcManager *flowcontrol.ClientManager // nil if our node is client only + fcCostStats *requestCostStats + defParams *flowcontrol.ServerParams + lesTopics []discv5.Topic + privateKey *ecdsa.PrivateKey + quitSync chan struct{} + onlyAnnounce bool } func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { quitSync := make(chan struct{}) - pm, err := NewProtocolManager(eth.BlockChain().Config(), light.DefaultServerIndexerConfig, false, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, nil, quitSync, new(sync.WaitGroup)) + pm, err := NewProtocolManager( + eth.BlockChain().Config(), + light.DefaultServerIndexerConfig, + false, + config.NetworkId, + eth.EventMux(), + eth.Engine(), + newPeerSet(), + eth.BlockChain(), + eth.TxPool(), + eth.ChainDb(), + nil, + nil, + nil, + quitSync, + new(sync.WaitGroup), + config.ULC) if err != nil { return nil, err } @@ -70,8 +87,9 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), nil, params.BloomBitsBlocks, params.BloomTrieFrequency), protocolManager: pm, }, - quitSync: quitSync, - lesTopics: lesTopics, + quitSync: quitSync, + lesTopics: lesTopics, + onlyAnnounce: config.OnlyAnnounce, } logger := log.New() @@ -289,10 +307,8 @@ func (s *requestCostStats) getCurrentList() RequestCostList { defer s.lock.Unlock() list := make(RequestCostList, len(reqList)) - //fmt.Println("RequestCostList") for idx, code := range reqList { b, m := s.stats[code].calc() - //fmt.Println(code, s.stats[code].cnt, b/1000000, m/1000000) if m < 0 { b += m m = 0 diff --git a/les/serverpool.go b/les/serverpool.go index 52b54b371f..3f4d0a1d9b 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -126,22 +126,22 @@ type serverPool struct { discNodes chan *enode.Node discLookups chan bool + trustedNodes map[enode.ID]*enode.Node entries map[enode.ID]*poolEntry timeout, enableRetry chan *poolEntry adjustStats chan poolStatAdjust - connCh chan *connReq - disconnCh chan *disconnReq - registerCh chan *registerReq - knownQueue, newQueue poolEntryQueue knownSelect, newSelect *weightedRandomSelect knownSelected, newSelected int fastDiscover bool + connCh chan *connReq + disconnCh chan *disconnReq + registerCh chan *registerReq } // newServerPool creates a new serverPool instance -func newServerPool(db ethdb.Database, quit chan struct{}, wg *sync.WaitGroup) *serverPool { +func newServerPool(db ethdb.Database, quit chan struct{}, wg *sync.WaitGroup, trustedNodes []string) *serverPool { pool := &serverPool{ db: db, quit: quit, @@ -156,7 +156,9 @@ func newServerPool(db ethdb.Database, quit chan struct{}, wg *sync.WaitGroup) *s knownSelect: newWeightedRandomSelect(), newSelect: newWeightedRandomSelect(), fastDiscover: true, + trustedNodes: parseTrustedNodes(trustedNodes), } + pool.knownQueue = newPoolEntryQueue(maxKnownEntries, pool.removeEntry) pool.newQueue = newPoolEntryQueue(maxNewEntries, pool.removeEntry) return pool @@ -168,6 +170,7 @@ func (pool *serverPool) start(server *p2p.Server, topic discv5.Topic) { pool.dbKey = append([]byte("serverPool/"), []byte(topic)...) pool.wg.Add(1) pool.loadNodes() + pool.connectToTrustedNodes() if pool.server.DiscV5 != nil { pool.discSetPeriod = make(chan time.Duration, 1) @@ -337,8 +340,10 @@ func (pool *serverPool) eventLoop() { } case node := <-pool.discNodes: - entry := pool.findOrNewNode(node) - pool.updateCheckDial(entry) + if pool.trustedNodes[node.ID()] == nil { + entry := pool.findOrNewNode(node) + pool.updateCheckDial(entry) + } case conv := <-pool.discLookups: if conv { @@ -355,29 +360,34 @@ func (pool *serverPool) eventLoop() { } case req := <-pool.connCh: - // Handle peer connection requests. - entry := pool.entries[req.p.ID()] - if entry == nil { - entry = pool.findOrNewNode(req.node) - } - if entry.state == psConnected || entry.state == psRegistered { + if pool.trustedNodes[req.p.ID()] != nil { + // ignore trusted nodes req.result <- nil - continue + } else { + // Handle peer connection requests. + entry := pool.entries[req.p.ID()] + if entry == nil { + entry = pool.findOrNewNode(req.node) + } + if entry.state == psConnected || entry.state == psRegistered { + req.result <- nil + continue + } + pool.connWg.Add(1) + entry.peer = req.p + entry.state = psConnected + addr := &poolEntryAddress{ + ip: req.node.IP(), + port: uint16(req.node.TCP()), + lastSeen: mclock.Now(), + } + entry.lastConnected = addr + entry.addr = make(map[string]*poolEntryAddress) + entry.addr[addr.strKey()] = addr + entry.addrSelect = *newWeightedRandomSelect() + entry.addrSelect.update(addr) + req.result <- entry } - pool.connWg.Add(1) - entry.peer = req.p - entry.state = psConnected - addr := &poolEntryAddress{ - ip: req.node.IP(), - port: uint16(req.node.TCP()), - lastSeen: mclock.Now(), - } - entry.lastConnected = addr - entry.addr = make(map[string]*poolEntryAddress) - entry.addr[addr.strKey()] = addr - entry.addrSelect = *newWeightedRandomSelect() - entry.addrSelect.update(addr) - req.result <- entry case req := <-pool.registerCh: // Handle peer registration requests. @@ -470,11 +480,42 @@ func (pool *serverPool) loadNodes() { "response", fmt.Sprintf("%v/%v", time.Duration(e.responseStats.avg), e.responseStats.weight), "timeout", fmt.Sprintf("%v/%v", e.timeoutStats.avg, e.timeoutStats.weight)) pool.entries[e.node.ID()] = e - pool.knownQueue.setLatest(e) - pool.knownSelect.update((*knownEntry)(e)) + if pool.trustedNodes[e.node.ID()] == nil { + pool.knownQueue.setLatest(e) + pool.knownSelect.update((*knownEntry)(e)) + } } } +// connectToTrustedNodes adds trusted server nodes as static trusted peers. +// +// Note: trusted nodes are not handled by the server pool logic, they are not +// added to either the known or new selection pools. They are connected/reconnected +// by p2p.Server whenever possible. +func (pool *serverPool) connectToTrustedNodes() { + //connect to trusted nodes + for _, node := range pool.trustedNodes { + pool.server.AddTrustedPeer(node) + pool.server.AddPeer(node) + log.Debug("Added trusted node", "id", node.ID().String()) + } +} + +// parseTrustedNodes returns valid and parsed enodes +func parseTrustedNodes(trustedNodes []string) map[enode.ID]*enode.Node { + nodes := make(map[enode.ID]*enode.Node) + + for _, node := range trustedNodes { + node, err := enode.ParseV4(node) + if err != nil { + log.Warn("Trusted node URL invalid", "enode", node, "err", err) + continue + } + nodes[node.ID()] = node + } + return nodes +} + // saveNodes saves known nodes and their statistics into the database. Nodes are // ordered from least to most recently connected. func (pool *serverPool) saveNodes() { diff --git a/les/txrelay.go b/les/txrelay.go index 7a02cc837e..6d22856f9f 100644 --- a/les/txrelay.go +++ b/les/txrelay.go @@ -121,7 +121,7 @@ func (self *LesTxRelay) send(txs types.Transactions, count int) { return peer.GetRequestCost(SendTxMsg, len(ll)) }, canSend: func(dp distPeer) bool { - return dp.(*peer) == pp + return !dp.(*peer).isOnlyAnnounce && dp.(*peer) == pp }, request: func(dp distPeer) func() { peer := dp.(*peer) diff --git a/les/ulc.go b/les/ulc.go new file mode 100644 index 0000000000..d9f7dc76c9 --- /dev/null +++ b/les/ulc.go @@ -0,0 +1,39 @@ +package les + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +type ulc struct { + trustedKeys map[string]struct{} + minTrustedFraction int +} + +func newULC(ulcConfig *eth.ULCConfig) *ulc { + if ulcConfig == nil { + return nil + } + + m := make(map[string]struct{}, len(ulcConfig.TrustedServers)) + for _, id := range ulcConfig.TrustedServers { + node, err := enode.ParseV4(id) + if err != nil { + fmt.Println("node:", id, " err:", err) + continue + } + m[node.ID().String()] = struct{}{} + } + + return &ulc{m, ulcConfig.MinTrustedFraction} +} + +func (u *ulc) isTrusted(p enode.ID) bool { + if u.trustedKeys == nil { + return false + } + _, ok := u.trustedKeys[p.String()] + return ok +} diff --git a/les/ulc_test.go b/les/ulc_test.go new file mode 100644 index 0000000000..3b95e6368e --- /dev/null +++ b/les/ulc_test.go @@ -0,0 +1,239 @@ +package les + +import ( + "fmt" + "reflect" + "testing" + "time" + + "net" + + "crypto/ecdsa" + "math/big" + + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/light" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +func TestULCSyncWithOnePeer(t *testing.T) { + f := newFullPeerPair(t, 1, 4, testChainGen) + ulcConfig := ð.ULCConfig{ + MinTrustedFraction: 100, + TrustedServers: []string{f.Node.String()}, + } + + l := newLightPeer(t, ulcConfig) + + if reflect.DeepEqual(f.PM.blockchain.CurrentHeader().Hash(), l.PM.blockchain.CurrentHeader().Hash()) { + t.Fatal("blocks are equal") + } + + _, _, err := connectPeers(f, l, 2) + if err != nil { + t.Fatal(err) + } + + l.PM.fetcher.lock.Lock() + l.PM.fetcher.nextRequest() + l.PM.fetcher.lock.Unlock() + + if !reflect.DeepEqual(f.PM.blockchain.CurrentHeader().Hash(), l.PM.blockchain.CurrentHeader().Hash()) { + t.Fatal("sync doesn't work") + } +} + +func TestULCReceiveAnnounce(t *testing.T) { + f := newFullPeerPair(t, 1, 4, testChainGen) + ulcConfig := ð.ULCConfig{ + MinTrustedFraction: 100, + TrustedServers: []string{f.Node.String()}, + } + + l := newLightPeer(t, ulcConfig) + fPeer, lPeer, err := connectPeers(f, l, 2) + if err != nil { + t.Fatal(err) + } + + l.PM.synchronise(fPeer) + + //check that the sync is finished correctly + if !reflect.DeepEqual(f.PM.blockchain.CurrentHeader().Hash(), l.PM.blockchain.CurrentHeader().Hash()) { + t.Fatal("sync doesn't work") + } + + l.PM.peers.lock.Lock() + if len(l.PM.peers.peers) == 0 { + t.Fatal("peer list should not be empty") + } + l.PM.peers.lock.Unlock() + + time.Sleep(time.Second) + //send a signed announce message(payload doesn't matter) + td := f.PM.blockchain.GetTd(l.PM.blockchain.CurrentHeader().Hash(), l.PM.blockchain.CurrentHeader().Number.Uint64()) + announce := announceData{ + Number: l.PM.blockchain.CurrentHeader().Number.Uint64() + 1, + Td: td.Add(td, big.NewInt(1)), + } + announce.sign(f.Key) + lPeer.SendAnnounce(announce) +} + +func TestULCShouldNotSyncWithTwoPeersOneHaveEmptyChain(t *testing.T) { + f1 := newFullPeerPair(t, 1, 4, testChainGen) + f2 := newFullPeerPair(t, 2, 0, nil) + ulcConf := &ulc{minTrustedFraction: 100, trustedKeys: make(map[string]struct{})} + ulcConf.trustedKeys[f1.Node.ID().String()] = struct{}{} + ulcConf.trustedKeys[f2.Node.ID().String()] = struct{}{} + ulcConfig := ð.ULCConfig{ + MinTrustedFraction: 100, + TrustedServers: []string{f1.Node.String(), f2.Node.String()}, + } + l := newLightPeer(t, ulcConfig) + l.PM.ulc.minTrustedFraction = 100 + + _, _, err := connectPeers(f1, l, 2) + if err != nil { + t.Fatal(err) + } + _, _, err = connectPeers(f2, l, 2) + if err != nil { + t.Fatal(err) + } + + l.PM.fetcher.lock.Lock() + l.PM.fetcher.nextRequest() + l.PM.fetcher.lock.Unlock() + + if reflect.DeepEqual(f2.PM.blockchain.CurrentHeader().Hash(), l.PM.blockchain.CurrentHeader().Hash()) { + t.Fatal("Incorrect hash: second peer has empty chain") + } +} + +func TestULCShouldNotSyncWithThreePeersOneHaveEmptyChain(t *testing.T) { + f1 := newFullPeerPair(t, 1, 3, testChainGen) + f2 := newFullPeerPair(t, 2, 4, testChainGen) + f3 := newFullPeerPair(t, 3, 0, nil) + + ulcConfig := ð.ULCConfig{ + MinTrustedFraction: 60, + TrustedServers: []string{f1.Node.String(), f2.Node.String(), f3.Node.String()}, + } + + l := newLightPeer(t, ulcConfig) + _, _, err := connectPeers(f1, l, 2) + if err != nil { + t.Fatal(err) + } + + _, _, err = connectPeers(f2, l, 2) + if err != nil { + t.Fatal(err) + } + + _, _, err = connectPeers(f3, l, 2) + if err != nil { + t.Fatal(err) + } + + l.PM.fetcher.lock.Lock() + l.PM.fetcher.nextRequest() + l.PM.fetcher.lock.Unlock() + + if !reflect.DeepEqual(f1.PM.blockchain.CurrentHeader().Hash(), l.PM.blockchain.CurrentHeader().Hash()) { + t.Fatal("Incorrect hash") + } +} + +type pairPeer struct { + Name string + Node *enode.Node + PM *ProtocolManager + Key *ecdsa.PrivateKey +} + +func connectPeers(full, light pairPeer, version int) (*peer, *peer, error) { + // Create a message pipe to communicate through + app, net := p2p.MsgPipe() + + peerLight := full.PM.newPeer(version, NetworkId, p2p.NewPeer(light.Node.ID(), light.Name, nil), net) + peerFull := light.PM.newPeer(version, NetworkId, p2p.NewPeer(full.Node.ID(), full.Name, nil), app) + + // Start the peerLight on a new thread + errc1 := make(chan error, 1) + errc2 := make(chan error, 1) + go func() { + select { + case light.PM.newPeerCh <- peerFull: + errc1 <- light.PM.handle(peerFull) + case <-light.PM.quitSync: + errc1 <- p2p.DiscQuitting + } + }() + go func() { + select { + case full.PM.newPeerCh <- peerLight: + errc2 <- full.PM.handle(peerLight) + case <-full.PM.quitSync: + errc2 <- p2p.DiscQuitting + } + }() + + select { + case <-time.After(time.Millisecond * 100): + case err := <-errc1: + return nil, nil, fmt.Errorf("peerLight handshake error: %v", err) + case err := <-errc2: + return nil, nil, fmt.Errorf("peerFull handshake error: %v", err) + } + + return peerFull, peerLight, nil +} + +// newFullPeerPair creates node with full sync mode +func newFullPeerPair(t *testing.T, index int, numberOfblocks int, chainGen func(int, *core.BlockGen)) pairPeer { + db := ethdb.NewMemDatabase() + + pmFull := newTestProtocolManagerMust(t, false, numberOfblocks, chainGen, nil, nil, db, nil) + + peerPairFull := pairPeer{ + Name: "full node", + PM: pmFull, + } + key, err := crypto.GenerateKey() + if err != nil { + t.Fatal("generate key err:", err) + } + peerPairFull.Key = key + peerPairFull.Node = enode.NewV4(&key.PublicKey, net.ParseIP("127.0.0.1"), 35000, 35000) + return peerPairFull +} + +// newLightPeer creates node with light sync mode +func newLightPeer(t *testing.T, ulcConfig *eth.ULCConfig) pairPeer { + peers := newPeerSet() + dist := newRequestDistributor(peers, make(chan struct{})) + rm := newRetrieveManager(peers, dist, nil) + ldb := ethdb.NewMemDatabase() + + odr := NewLesOdr(ldb, light.DefaultClientIndexerConfig, rm) + + pmLight := newTestProtocolManagerMust(t, true, 0, nil, odr, peers, ldb, ulcConfig) + peerPairLight := pairPeer{ + Name: "ulc node", + PM: pmLight, + } + + key, err := crypto.GenerateKey() + if err != nil { + t.Fatal("generate key err:", err) + } + peerPairLight.Key = key + peerPairLight.Node = enode.NewV4(&key.PublicKey, net.IP{}, 35000, 35000) + return peerPairLight +} diff --git a/light/lightchain.go b/light/lightchain.go index 8e2734c2d9..3691f1304f 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -35,7 +35,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "github.com/hashicorp/golang-lru" + lru "github.com/hashicorp/golang-lru" ) var ( @@ -50,6 +50,7 @@ type LightChain struct { hc *core.HeaderChain indexerConfig *IndexerConfig chainDb ethdb.Database + engine consensus.Engine odr OdrBackend chainFeed event.Feed chainSideFeed event.Feed @@ -57,20 +58,18 @@ type LightChain struct { scope event.SubscriptionScope genesisBlock *types.Block - mu sync.RWMutex - chainmu sync.RWMutex - bodyCache *lru.Cache // Cache for the most recent block bodies bodyRLPCache *lru.Cache // Cache for the most recent block bodies in RLP encoded format blockCache *lru.Cache // Cache for the most recent entire blocks + chainmu sync.RWMutex // protects header inserts quit chan struct{} - running int32 // running must be called automically - // procInterrupt must be atomically called - procInterrupt int32 // interrupt signaler for block processing - wg sync.WaitGroup + wg sync.WaitGroup - engine consensus.Engine + // Atomic boolean switches: + running int32 // whether LightChain is running or stopped + procInterrupt int32 // interrupts chain insert + disableCheckFreq int32 // disables header verification } // NewLightChain returns a fully initialised light chain using information @@ -165,8 +164,8 @@ func (self *LightChain) loadLastState() error { // SetHead rewinds the local chain to a new head. Everything above the new // head will be deleted and the new one set. func (bc *LightChain) SetHead(head uint64) { - bc.mu.Lock() - defer bc.mu.Unlock() + bc.chainmu.Lock() + defer bc.chainmu.Unlock() bc.hc.SetHead(head, nil) bc.loadLastState() @@ -188,8 +187,8 @@ func (bc *LightChain) ResetWithGenesisBlock(genesis *types.Block) { // Dump the entire block chain and purge the caches bc.SetHead(0) - bc.mu.Lock() - defer bc.mu.Unlock() + bc.chainmu.Lock() + defer bc.chainmu.Unlock() // Prepare the genesis block and reinitialise the chain rawdb.WriteTd(bc.chainDb, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()) @@ -315,8 +314,8 @@ func (bc *LightChain) Stop() { // Rollback is designed to remove a chain of links from the database that aren't // certain enough to be valid. func (self *LightChain) Rollback(chain []common.Hash) { - self.mu.Lock() - defer self.mu.Unlock() + self.chainmu.Lock() + defer self.chainmu.Unlock() for i := len(chain) - 1; i >= 0; i-- { hash := chain[i] @@ -355,6 +354,9 @@ func (self *LightChain) postChainEvents(events []interface{}) { // In the case of a light chain, InsertHeaderChain also creates and posts light // chain events when necessary. func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) { + if atomic.LoadInt32(&self.disableCheckFreq) == 1 { + checkFreq = 0 + } start := time.Now() if i, err := self.hc.ValidateHeaderChain(chain, checkFreq); err != nil { return i, err @@ -362,19 +364,13 @@ func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) // Make sure only one thread manipulates the chain at once self.chainmu.Lock() - defer func() { - self.chainmu.Unlock() - time.Sleep(time.Millisecond * 10) // ugly hack; do not hog chain lock in case syncing is CPU-limited by validation - }() + defer self.chainmu.Unlock() self.wg.Add(1) defer self.wg.Done() var events []interface{} whFunc := func(header *types.Header) error { - self.mu.Lock() - defer self.mu.Unlock() - status, err := self.hc.WriteHeader(header) switch status { @@ -441,8 +437,8 @@ func (self *LightChain) GetBlockHashesFromHash(hash common.Hash, max uint64) []c // // Note: ancestor == 0 returns the same block, 1 returns its parent and so on. func (bc *LightChain) GetAncestor(hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) { - bc.chainmu.Lock() - defer bc.chainmu.Unlock() + bc.chainmu.RLock() + defer bc.chainmu.RUnlock() return bc.hc.GetAncestor(hash, number, ancestor, maxNonCanonical) } @@ -483,8 +479,8 @@ func (self *LightChain) SyncCht(ctx context.Context) bool { } // Retrieve the latest useful header and update to it if header, err := GetHeaderByNumber(ctx, self.odr, latest); header != nil && err == nil { - self.mu.Lock() - defer self.mu.Unlock() + self.chainmu.Lock() + defer self.chainmu.Unlock() // Ensure the chain didn't move past the latest block while retrieving it if self.hc.CurrentHeader().Number.Uint64() < header.Number.Uint64() { @@ -533,3 +529,13 @@ func (self *LightChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscri func (self *LightChain) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return self.scope.Track(new(event.Feed).Subscribe(ch)) } + +// DisableCheckFreq disables header validation. This is used for ultralight mode. +func (self *LightChain) DisableCheckFreq() { + atomic.StoreInt32(&self.disableCheckFreq, 1) +} + +// EnableCheckFreq enables header validation. +func (self *LightChain) EnableCheckFreq() { + atomic.StoreInt32(&self.disableCheckFreq, 0) +} diff --git a/light/lightchain_test.go b/light/lightchain_test.go index d45c0656d9..374cb53198 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -122,10 +122,10 @@ func testHeaderChainImport(chain []*types.Header, lightchain *LightChain) error return err } // Manually insert the header into the database, but don't reorganize (allows subsequent testing) - lightchain.mu.Lock() + lightchain.chainmu.Lock() rawdb.WriteTd(lightchain.chainDb, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, lightchain.GetTdByHash(header.ParentHash))) rawdb.WriteHeader(lightchain.chainDb, header) - lightchain.mu.Unlock() + lightchain.chainmu.Unlock() } return nil } diff --git a/metrics/registry.go b/metrics/registry.go index c1cf7906ce..c5435adf24 100644 --- a/metrics/registry.go +++ b/metrics/registry.go @@ -312,8 +312,9 @@ func (r *PrefixedRegistry) UnregisterAll() { } var ( - DefaultRegistry = NewRegistry() - EphemeralRegistry = NewRegistry() + DefaultRegistry = NewRegistry() + EphemeralRegistry = NewRegistry() + AccountingRegistry = NewRegistry() // registry used in swarm ) // Call the given function for each registered metric. diff --git a/miner/worker_test.go b/miner/worker_test.go index 7c8f167a12..bfa9cbd21d 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -224,7 +224,7 @@ func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consens for i := 0; i < 2; i += 1 { select { case <-taskCh: - case <-time.NewTimer(time.Second).C: + case <-time.NewTimer(2 * time.Second).C: t.Error("new task timeout") } } diff --git a/mobile/geth.go b/mobile/geth.go index e3e2e905da..781f42f708 100644 --- a/mobile/geth.go +++ b/mobile/geth.go @@ -76,6 +76,9 @@ type NodeConfig struct { // Listening address of pprof server. PprofAddress string + + // Ultra Light client options + ULC *eth.ULCConfig } // defaultNodeConfig contains the default node configuration values to use if all @@ -131,6 +134,7 @@ func NewNode(datadir string, config *NodeConfig) (stack *Node, _ error) { MaxPeers: config.MaxPeers, }, } + rawStack, err := node.New(nodeConf) if err != nil { return nil, err diff --git a/node/config.go b/node/config.go index 7b32a59087..99f3258403 100644 --- a/node/config.go +++ b/node/config.go @@ -102,6 +102,29 @@ type Config struct { // for ephemeral nodes). HTTPPort int `toml:",omitempty"` + // GraphQLHost is the host interface on which to start the GraphQL server. If this + // field is empty, no GraphQL API endpoint will be started. + GraphQLHost string `toml:",omitempty"` + + // GraphQLPort is the TCP port number on which to start the GraphQL server. The + // default zero value is/ valid and will pick a port number randomly (useful + // for ephemeral nodes). + GraphQLPort int `toml:",omitempty"` + + // GraphQLCors is the Cross-Origin Resource Sharing header to send to requesting + // clients. Please be aware that CORS is a browser enforced security, it's fully + // useless for custom HTTP clients. + GraphQLCors []string `toml:",omitempty"` + + // GraphQLVirtualHosts is the list of virtual hostnames which are allowed on incoming requests. + // This is by default {'localhost'}. Using this prevents attacks like + // DNS rebinding, which bypasses SOP by simply masquerading as being within the same + // origin. These attacks do not utilize CORS, since they are not cross-domain. + // By explicitly checking the Host-header, the server will not allow requests + // made against the server with a malicious host domain. + // Requests using ip address directly are not affected + GraphQLVirtualHosts []string `toml:",omitempty"` + // HTTPCors is the Cross-Origin Resource Sharing header to send to requesting // clients. Please be aware that CORS is a browser enforced security, it's fully // useless for custom HTTP clients. @@ -213,6 +236,15 @@ func (c *Config) HTTPEndpoint() string { return fmt.Sprintf("%s:%d", c.HTTPHost, c.HTTPPort) } +// GraphQLEndpoint resolves a GraphQL endpoint based on the configured host interface +// and port parameters. +func (c *Config) GraphQLEndpoint() string { + if c.GraphQLHost == "" { + return "" + } + return fmt.Sprintf("%s:%d", c.GraphQLHost, c.GraphQLPort) +} + // DefaultHTTPEndpoint returns the HTTP endpoint used by default. func DefaultHTTPEndpoint() string { config := &Config{HTTPHost: DefaultHTTPHost, HTTPPort: DefaultHTTPPort} diff --git a/node/defaults.go b/node/defaults.go index c1376dba0f..cea4997cb4 100644 --- a/node/defaults.go +++ b/node/defaults.go @@ -28,10 +28,12 @@ import ( ) const ( - DefaultHTTPHost = "localhost" // Default host interface for the HTTP RPC server - DefaultHTTPPort = 8545 // Default TCP port for the HTTP RPC server - DefaultWSHost = "localhost" // Default host interface for the websocket RPC server - DefaultWSPort = 8546 // Default TCP port for the websocket RPC server + DefaultHTTPHost = "localhost" // Default host interface for the HTTP RPC server + DefaultHTTPPort = 8545 // Default TCP port for the HTTP RPC server + DefaultWSHost = "localhost" // Default host interface for the websocket RPC server + DefaultWSPort = 8546 // Default TCP port for the websocket RPC server + DefaultGraphQLHost = "localhost" // Default host interface for the GraphQL server + DefaultGraphQLPort = 8547 // Default TCP port for the GraphQL server ) // DefaultConfig contains reasonable default settings. diff --git a/p2p/protocols/accounting.go b/p2p/protocols/accounting.go index bdc490e591..558247254a 100644 --- a/p2p/protocols/accounting.go +++ b/p2p/protocols/accounting.go @@ -27,23 +27,21 @@ var ( // All metrics are cumulative // total amount of units credited - mBalanceCredit metrics.Counter + mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", metrics.AccountingRegistry) // total amount of units debited - mBalanceDebit metrics.Counter + mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", metrics.AccountingRegistry) // total amount of bytes credited - mBytesCredit metrics.Counter + mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", metrics.AccountingRegistry) // total amount of bytes debited - mBytesDebit metrics.Counter + mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", metrics.AccountingRegistry) // total amount of credited messages - mMsgCredit metrics.Counter + mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", metrics.AccountingRegistry) // total amount of debited messages - mMsgDebit metrics.Counter + mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", metrics.AccountingRegistry) // how many times local node had to drop remote peers - mPeerDrops metrics.Counter + mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", metrics.AccountingRegistry) // how many times local node overdrafted and dropped - mSelfDrops metrics.Counter - - MetricsRegistry metrics.Registry + mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", metrics.AccountingRegistry) ) // Prices defines how prices are being passed on to the accounting instance @@ -110,24 +108,13 @@ func NewAccounting(balance Balance, po Prices) *Accounting { return ah } -// SetupAccountingMetrics creates a separate registry for p2p accounting metrics; +// SetupAccountingMetrics uses a separate registry for p2p accounting metrics; // this registry should be independent of any other metrics as it persists at different endpoints. -// It also instantiates the given metrics and starts the persisting go-routine which +// It also starts the persisting go-routine which // at the passed interval writes the metrics to a LevelDB func SetupAccountingMetrics(reportInterval time.Duration, path string) *AccountingMetrics { - // create an empty registry - MetricsRegistry = metrics.NewRegistry() - // instantiate the metrics - mBalanceCredit = metrics.NewRegisteredCounterForced("account.balance.credit", MetricsRegistry) - mBalanceDebit = metrics.NewRegisteredCounterForced("account.balance.debit", MetricsRegistry) - mBytesCredit = metrics.NewRegisteredCounterForced("account.bytes.credit", MetricsRegistry) - mBytesDebit = metrics.NewRegisteredCounterForced("account.bytes.debit", MetricsRegistry) - mMsgCredit = metrics.NewRegisteredCounterForced("account.msg.credit", MetricsRegistry) - mMsgDebit = metrics.NewRegisteredCounterForced("account.msg.debit", MetricsRegistry) - mPeerDrops = metrics.NewRegisteredCounterForced("account.peerdrops", MetricsRegistry) - mSelfDrops = metrics.NewRegisteredCounterForced("account.selfdrops", MetricsRegistry) // create the DB and start persisting - return NewAccountingMetrics(MetricsRegistry, reportInterval, path) + return NewAccountingMetrics(metrics.AccountingRegistry, reportInterval, path) } // Send takes a peer, a size and a msg and diff --git a/p2p/protocols/protocol.go b/p2p/protocols/protocol.go index b16720dd39..bf879b9851 100644 --- a/p2p/protocols/protocol.go +++ b/p2p/protocols/protocol.go @@ -423,3 +423,17 @@ func (p *Peer) Handshake(ctx context.Context, hs interface{}, verify func(interf } return rhs, nil } + +// HasCap returns true if Peer has a capability +// with provided name. +func (p *Peer) HasCap(capName string) (yes bool) { + if p == nil || p.Peer == nil { + return false + } + for _, c := range p.Caps() { + if c.Name == capName { + return true + } + } + return false +} diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index a26222cd8b..cfdd05f4c9 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -142,9 +142,9 @@ func newProtocol(pp *p2ptest.TestPeerPool) func(*p2p.Peer, p2p.MsgReadWriter) er } } -func protocolTester(t *testing.T, pp *p2ptest.TestPeerPool) *p2ptest.ProtocolTester { +func protocolTester(pp *p2ptest.TestPeerPool) *p2ptest.ProtocolTester { conf := adapters.RandomNodeConfig() - return p2ptest.NewProtocolTester(t, conf.ID, 2, newProtocol(pp)) + return p2ptest.NewProtocolTester(conf.ID, 2, newProtocol(pp)) } func protoHandshakeExchange(id enode.ID, proto *protoHandshake) []p2ptest.Exchange { @@ -173,7 +173,7 @@ func protoHandshakeExchange(id enode.ID, proto *protoHandshake) []p2ptest.Exchan func runProtoHandshake(t *testing.T, proto *protoHandshake, errs ...error) { pp := p2ptest.NewTestPeerPool() - s := protocolTester(t, pp) + s := protocolTester(pp) // TODO: make this more than one handshake node := s.Nodes[0] if err := s.TestExchanges(protoHandshakeExchange(node.ID(), proto)...); err != nil { @@ -250,7 +250,7 @@ func TestProtocolHook(t *testing.T) { } conf := adapters.RandomNodeConfig() - tester := p2ptest.NewProtocolTester(t, conf.ID, 2, runFunc) + tester := p2ptest.NewProtocolTester(conf.ID, 2, runFunc) err := tester.TestExchanges(p2ptest.Exchange{ Expects: []p2ptest.Expect{ { @@ -269,8 +269,11 @@ func TestProtocolHook(t *testing.T) { if !testHook.send { t.Fatal("Expected a send message, but it is not") } - if testHook.peer == nil || testHook.peer.ID() != tester.Nodes[0].ID() { - t.Fatal("Expected peer ID to be set correctly, but it is not") + if testHook.peer == nil { + t.Fatal("Expected peer to be set, is nil") + } + if peerId := testHook.peer.ID(); peerId != tester.Nodes[0].ID() && peerId != tester.Nodes[1].ID() { + t.Fatalf("Expected peer ID to be set correctly, but it is not (got %v, exp %v or %v", peerId, tester.Nodes[0].ID(), tester.Nodes[1].ID()) } if testHook.size != 11 { //11 is the length of the encoded message t.Fatalf("Expected size to be %d, but it is %d ", 1, testHook.size) @@ -389,7 +392,7 @@ func moduleHandshakeExchange(id enode.ID, resp uint) []p2ptest.Exchange { func runModuleHandshake(t *testing.T, resp uint, errs ...error) { pp := p2ptest.NewTestPeerPool() - s := protocolTester(t, pp) + s := protocolTester(pp) node := s.Nodes[0] if err := s.TestExchanges(protoHandshakeExchange(node.ID(), &protoHandshake{42, "420"})...); err != nil { t.Fatal(err) @@ -469,7 +472,7 @@ func testMultiPeerSetup(a, b enode.ID) []p2ptest.Exchange { func runMultiplePeers(t *testing.T, peer int, errs ...error) { pp := p2ptest.NewTestPeerPool() - s := protocolTester(t, pp) + s := protocolTester(pp) if err := s.TestExchanges(testMultiPeerSetup(s.Nodes[0].ID(), s.Nodes[1].ID())...); err != nil { t.Fatal(err) diff --git a/p2p/protocols/reporter_test.go b/p2p/protocols/reporter_test.go index b9f06e6744..9b0da09b79 100644 --- a/p2p/protocols/reporter_test.go +++ b/p2p/protocols/reporter_test.go @@ -39,25 +39,31 @@ func TestReporter(t *testing.T) { //setup the metrics log.Debug("Setting up metrics first time") - reportInterval := 5 * time.Millisecond + reportInterval := 2 * time.Millisecond metrics := SetupAccountingMetrics(reportInterval, filepath.Join(dir, "test.db")) log.Debug("Done.") - //do some metrics + //change metrics mBalanceCredit.Inc(12) mBytesCredit.Inc(34) mMsgDebit.Inc(9) + //store expected metrics + expectedBalanceCredit := mBalanceCredit.Count() + expectedBytesCredit := mBytesCredit.Count() + expectedMsgDebit := mMsgDebit.Count() + //give the reporter time to write the metrics to DB time.Sleep(20 * time.Millisecond) - //set the metrics to nil - this effectively simulates the node having shut down... - mBalanceCredit = nil - mBytesCredit = nil - mMsgDebit = nil //close the DB also, or we can't create a new one metrics.Close() + //clear the metrics - this effectively simulates the node having shut down... + mBalanceCredit.Clear() + mBytesCredit.Clear() + mMsgDebit.Clear() + //setup the metrics again log.Debug("Setting up metrics second time") metrics = SetupAccountingMetrics(reportInterval, filepath.Join(dir, "test.db")) @@ -65,13 +71,13 @@ func TestReporter(t *testing.T) { log.Debug("Done.") //now check the metrics, they should have the same value as before "shutdown" - if mBalanceCredit.Count() != 12 { - t.Fatalf("Expected counter to be %d, but is %d", 12, mBalanceCredit.Count()) + if mBalanceCredit.Count() != expectedBalanceCredit { + t.Fatalf("Expected counter to be %d, but is %d", expectedBalanceCredit, mBalanceCredit.Count()) } - if mBytesCredit.Count() != 34 { - t.Fatalf("Expected counter to be %d, but is %d", 23, mBytesCredit.Count()) + if mBytesCredit.Count() != expectedBytesCredit { + t.Fatalf("Expected counter to be %d, but is %d", expectedBytesCredit, mBytesCredit.Count()) } - if mMsgDebit.Count() != 9 { - t.Fatalf("Expected counter to be %d, but is %d", 9, mMsgDebit.Count()) + if mMsgDebit.Count() != expectedMsgDebit { + t.Fatalf("Expected counter to be %d, but is %d", expectedMsgDebit, mMsgDebit.Count()) } } diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 86f7dc9bef..6edcefc532 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -168,24 +168,28 @@ func (net *Network) Start(id enode.ID) error { // snapshots func (net *Network) startWithSnapshots(id enode.ID, snapshots map[string][]byte) error { net.lock.Lock() - defer net.lock.Unlock() node := net.getNode(id) if node == nil { + net.lock.Unlock() return fmt.Errorf("node %v does not exist", id) } if node.Up { + net.lock.Unlock() return fmt.Errorf("node %v already up", id) } log.Trace("Starting node", "id", id, "adapter", net.nodeAdapter.Name()) if err := node.Start(snapshots); err != nil { + net.lock.Unlock() log.Warn("Node startup failed", "id", id, "err", err) return err } node.Up = true log.Info("Started node", "id", id) + ev := NewEvent(node) + net.lock.Unlock() - net.events.Send(NewEvent(node)) + net.events.Send(ev) // subscribe to peer events client, err := node.Client() @@ -210,12 +214,14 @@ func (net *Network) watchPeerEvents(id enode.ID, events chan *p2p.PeerEvent, sub // assume the node is now down net.lock.Lock() defer net.lock.Unlock() + node := net.getNode(id) if node == nil { return } node.Up = false - net.events.Send(NewEvent(node)) + ev := NewEvent(node) + net.events.Send(ev) }() for { select { @@ -254,9 +260,11 @@ func (net *Network) Stop(id enode.ID) error { net.lock.Lock() node := net.getNode(id) if node == nil { + net.lock.Unlock() return fmt.Errorf("node %v does not exist", id) } if !node.Up { + net.lock.Unlock() return fmt.Errorf("node %v already down", id) } node.Up = false @@ -270,7 +278,10 @@ func (net *Network) Stop(id enode.ID) error { return err } log.Info("Stopped node", "id", id, "err", err) - net.events.Send(ControlEvent(node)) + net.lock.Lock() + ev := ControlEvent(node) + net.lock.Unlock() + net.events.Send(ev) return nil } diff --git a/p2p/testing/protocoltester.go b/p2p/testing/protocoltester.go index afc03b009f..cbd8ce6fe5 100644 --- a/p2p/testing/protocoltester.go +++ b/p2p/testing/protocoltester.go @@ -30,7 +30,6 @@ import ( "io/ioutil" "strings" "sync" - "testing" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -52,7 +51,7 @@ type ProtocolTester struct { // NewProtocolTester constructs a new ProtocolTester // it takes as argument the pivot node id, the number of dummy peers and the // protocol run function called on a peer connection by the p2p server -func NewProtocolTester(t *testing.T, id enode.ID, n int, run func(*p2p.Peer, p2p.MsgReadWriter) error) *ProtocolTester { +func NewProtocolTester(id enode.ID, n int, run func(*p2p.Peer, p2p.MsgReadWriter) error) *ProtocolTester { services := adapters.Services{ "test": func(ctx *adapters.ServiceContext) (node.Service, error) { return &testNode{run}, nil diff --git a/params/config.go b/params/config.go index fefc161061..cadf4e9330 100644 --- a/params/config.go +++ b/params/config.go @@ -42,7 +42,8 @@ var ( EIP155Block: big.NewInt(2675000), EIP158Block: big.NewInt(2675000), ByzantiumBlock: big.NewInt(4370000), - ConstantinopleBlock: nil, + ConstantinopleBlock: big.NewInt(7280000), + PetersburgBlock: big.NewInt(7280000), Ethash: new(EthashConfig), } @@ -67,6 +68,7 @@ var ( EIP158Block: big.NewInt(10), ByzantiumBlock: big.NewInt(1700000), ConstantinopleBlock: big.NewInt(4230000), + PetersburgBlock: big.NewInt(4939394), Ethash: new(EthashConfig), } @@ -91,6 +93,7 @@ var ( EIP158Block: big.NewInt(3), ByzantiumBlock: big.NewInt(1035301), ConstantinopleBlock: big.NewInt(3660663), + PetersburgBlock: big.NewInt(9999999), //TODO! Insert Rinkeby block number Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -111,16 +114,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -158,6 +161,7 @@ type ChainConfig struct { ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = already on byzantium) ConstantinopleBlock *big.Int `json:"constantinopleBlock,omitempty"` // Constantinople switch block (nil = no fork, 0 = already activated) + PetersburgBlock *big.Int `json:"petersburgBlock,omitempty"` // Petersburg switch block (nil = same as Constantinople) EWASMBlock *big.Int `json:"ewasmBlock,omitempty"` // EWASM switch block (nil = no fork, 0 = already activated) // Various consensus engines @@ -195,7 +199,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v ConstantinopleFix: %v Engine: %v}", c.ChainID, c.HomesteadBlock, c.DAOForkBlock, @@ -205,6 +209,7 @@ func (c *ChainConfig) String() string { c.EIP158Block, c.ByzantiumBlock, c.ConstantinopleBlock, + c.PetersburgBlock, engine, ) } @@ -244,6 +249,13 @@ func (c *ChainConfig) IsConstantinople(num *big.Int) bool { return isForked(c.ConstantinopleBlock, num) } +// IsPetersburg returns whether num is either +// - equal to or greater than the PetersburgBlock fork block, +// - OR is nil, and Constantinople is active +func (c *ChainConfig) IsPetersburg(num *big.Int) bool { + return isForked(c.PetersburgBlock, num) || c.PetersburgBlock == nil && isForked(c.ConstantinopleBlock, num) +} + // IsEWASM returns whether num represents a block number after the EWASM fork func (c *ChainConfig) IsEWASM(num *big.Int) bool { return isForked(c.EWASMBlock, num) @@ -314,6 +326,9 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.ConstantinopleBlock, newcfg.ConstantinopleBlock, head) { return newCompatError("Constantinople fork block", c.ConstantinopleBlock, newcfg.ConstantinopleBlock) } + if isForkIncompatible(c.PetersburgBlock, newcfg.PetersburgBlock, head) { + return newCompatError("ConstantinopleFix fork block", c.PetersburgBlock, newcfg.PetersburgBlock) + } if isForkIncompatible(c.EWASMBlock, newcfg.EWASMBlock, head) { return newCompatError("ewasm fork block", c.EWASMBlock, newcfg.EWASMBlock) } @@ -381,9 +396,9 @@ func (err *ConfigCompatError) Error() string { // Rules is a one time interface meaning that it shouldn't be used in between transition // phases. type Rules struct { - ChainID *big.Int - IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool - IsByzantium, IsConstantinople bool + ChainID *big.Int + IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool + IsByzantium, IsConstantinople, IsPetersburg bool } // Rules ensures c's ChainID is not nil. @@ -400,5 +415,6 @@ func (c *ChainConfig) Rules(num *big.Int) Rules { IsEIP158: c.IsEIP158(num), IsByzantium: c.IsByzantium(num), IsConstantinople: c.IsConstantinople(num), + IsPetersburg: c.IsPetersburg(num), } } diff --git a/params/protocol_params.go b/params/protocol_params.go index c8b6609afb..14750f6a1a 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -52,7 +52,7 @@ const ( NetSstoreResetRefund uint64 = 4800 // Once per SSTORE operation for resetting to the original non-zero value NetSstoreResetClearRefund uint64 = 19800 // Once per SSTORE operation for resetting to the original zero value - JumpdestGas uint64 = 1 // Refunded gas, once per SSTORE operation if the zeroness changes to zero. + JumpdestGas uint64 = 1 // Once per JUMPDEST operation. EpochDuration uint64 = 30000 // Duration between proof-of-work epochs. CallGas uint64 = 40 // Once per CALL operation & message call transaction. CreateDataGas uint64 = 200 // diff --git a/rpc/http.go b/rpc/http.go index 674166fb3a..59dd5eebc6 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -198,7 +198,7 @@ func (t *httpReadWriteNopCloser) Close() error { // NewHTTPServer creates a new HTTP RPC server around an API provider. // // Deprecated: Server implements http.Handler -func NewHTTPServer(cors []string, vhosts []string, timeouts HTTPTimeouts, srv *Server) *http.Server { +func NewHTTPServer(cors []string, vhosts []string, timeouts HTTPTimeouts, srv http.Handler) *http.Server { // Wrap the CORS-handler within a host-handler handler := newCorsHandler(srv, cors) handler = newVHostHandler(vhosts, handler) @@ -284,7 +284,7 @@ func validateRequest(r *http.Request) (int, error) { return http.StatusUnsupportedMediaType, err } -func newCorsHandler(srv *Server, allowedOrigins []string) http.Handler { +func newCorsHandler(srv http.Handler, allowedOrigins []string) http.Handler { // disable CORS support if user has not specified a custom CORS configuration if len(allowedOrigins) == 0 { return srv diff --git a/swarm/api/config.go b/swarm/api/config.go index be73854087..54dd67ba83 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -66,6 +66,7 @@ type Config struct { DeliverySkipCheck bool MaxStreamPeerServers int LightNodeEnabled bool + BootnodeMode bool SyncUpdateDelay time.Duration SwapAPI string Cors string diff --git a/swarm/metrics/flags.go b/swarm/metrics/flags.go index 7c12120a60..d348dc3e4d 100644 --- a/swarm/metrics/flags.go +++ b/swarm/metrics/flags.go @@ -23,7 +23,7 @@ import ( gethmetrics "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/metrics/influxdb" "github.com/ethereum/go-ethereum/swarm/log" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) var ( @@ -31,6 +31,10 @@ var ( Name: "metrics.influxdb.export", Usage: "Enable metrics export/push to an external InfluxDB database", } + MetricsEnableInfluxDBAccountingExportFlag = cli.BoolFlag{ + Name: "metrics.influxdb.accounting", + Usage: "Enable accounting metrics export/push to an external InfluxDB database", + } MetricsInfluxDBEndpointFlag = cli.StringFlag{ Name: "metrics.influxdb.endpoint", Usage: "Metrics InfluxDB endpoint", @@ -51,14 +55,14 @@ var ( Usage: "Metrics InfluxDB password", Value: "", } - // The `host` tag is part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. - // It is used so that we can group all nodes and average a measurement across all of them, but also so - // that we can select a specific node and inspect its measurements. + // Tags are part of every measurement sent to InfluxDB. Queries on tags are faster in InfluxDB. + // For example `host` tag could be used so that we can group all nodes and average a measurement + // across all of them, but also so that we can select a specific node and inspect its measurements. // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key - MetricsInfluxDBHostTagFlag = cli.StringFlag{ - Name: "metrics.influxdb.host.tag", - Usage: "Metrics InfluxDB `host` tag attached to all measurements", - Value: "localhost", + MetricsInfluxDBTagsFlag = cli.StringFlag{ + Name: "metrics.influxdb.tags", + Usage: "Comma-separated InfluxDB tags (key/values) attached to all measurements", + Value: "host=localhost", } ) @@ -66,33 +70,39 @@ var ( var Flags = []cli.Flag{ utils.MetricsEnabledFlag, MetricsEnableInfluxDBExportFlag, + MetricsEnableInfluxDBAccountingExportFlag, MetricsInfluxDBEndpointFlag, MetricsInfluxDBDatabaseFlag, MetricsInfluxDBUsernameFlag, MetricsInfluxDBPasswordFlag, - MetricsInfluxDBHostTagFlag, + MetricsInfluxDBTagsFlag, } func Setup(ctx *cli.Context) { if gethmetrics.Enabled { log.Info("Enabling swarm metrics collection") var ( - enableExport = ctx.GlobalBool(MetricsEnableInfluxDBExportFlag.Name) - endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) - database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) - username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) - password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(MetricsInfluxDBHostTagFlag.Name) + endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) + enableExport = ctx.GlobalBool(MetricsEnableInfluxDBExportFlag.Name) + enableAccountingExport = ctx.GlobalBool(MetricsEnableInfluxDBAccountingExportFlag.Name) ) // Start system runtime metrics collection go gethmetrics.CollectProcessMetrics(2 * time.Second) + tagsMap := utils.SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) + if enableExport { log.Info("Enabling swarm metrics export to InfluxDB") - go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "swarm.", map[string]string{ - "host": hosttag, - }) + go influxdb.InfluxDBWithTags(gethmetrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "swarm.", tagsMap) + } + + if enableAccountingExport { + log.Info("Exporting swarm accounting metrics to InfluxDB") + go influxdb.InfluxDBWithTags(gethmetrics.AccountingRegistry, 10*time.Second, endpoint, database, username, password, "accounting.", tagsMap) } } } diff --git a/swarm/network/fetcher.go b/swarm/network/fetcher.go index 6aed57e225..3043f64758 100644 --- a/swarm/network/fetcher.go +++ b/swarm/network/fetcher.go @@ -26,20 +26,23 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage" ) -var searchTimeout = 1 * time.Second +const ( + defaultSearchTimeout = 1 * time.Second + // maximum number of forwarded requests (hops), to make sure requests are not + // forwarded forever in peer loops + maxHopCount uint8 = 20 +) // Time to consider peer to be skipped. // Also used in stream delivery. var RequestTimeout = 10 * time.Second -var maxHopCount uint8 = 20 // maximum number of forwarded requests (hops), to make sure requests are not forwarded forever in peer loops - type RequestFunc func(context.Context, *Request) (*enode.ID, chan struct{}, error) // Fetcher is created when a chunk is not found locally. It starts a request handler loop once and // keeps it alive until all active requests are completed. This can happen: // 1. either because the chunk is delivered -// 2. or becuse the requestor cancelled/timed out +// 2. or because the requester cancelled/timed out // Fetcher self destroys itself after it is completed. // TODO: cancel all forward requests after termination type Fetcher struct { @@ -47,6 +50,7 @@ type Fetcher struct { addr storage.Address // the address of the chunk to be fetched offerC chan *enode.ID // channel of sources (peer node id strings) requestC chan uint8 // channel for incoming requests (with the hopCount value in it) + searchTimeout time.Duration skipCheck bool } @@ -79,7 +83,7 @@ func (r *Request) SkipPeer(nodeID string) bool { } t, ok := val.(time.Time) if ok && time.Now().After(t.Add(RequestTimeout)) { - // deadine expired + // deadline expired r.peersToSkip.Delete(nodeID) return false } @@ -100,9 +104,10 @@ func NewFetcherFactory(request RequestFunc, skipCheck bool) *FetcherFactory { } } -// New contructs a new Fetcher, for the given chunk. All peers in peersToSkip are not requested to -// deliver the given chunk. peersToSkip should always contain the peers which are actively requesting -// this chunk, to make sure we don't request back the chunks from them. +// New constructs a new Fetcher, for the given chunk. All peers in peersToSkip +// are not requested to deliver the given chunk. peersToSkip should always +// contain the peers which are actively requesting this chunk, to make sure we +// don't request back the chunks from them. // The created Fetcher is started and returned. func (f *FetcherFactory) New(ctx context.Context, source storage.Address, peersToSkip *sync.Map) storage.NetFetcher { fetcher := NewFetcher(source, f.request, f.skipCheck) @@ -117,6 +122,7 @@ func NewFetcher(addr storage.Address, rf RequestFunc, skipCheck bool) *Fetcher { protoRequestFunc: rf, offerC: make(chan *enode.ID), requestC: make(chan uint8), + searchTimeout: defaultSearchTimeout, skipCheck: skipCheck, } } @@ -176,7 +182,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { // loop that keeps the fetching process alive // after every request a timer is set. If this goes off we request again from another peer // note that the previous request is still alive and has the chance to deliver, so - // rerequesting extends the search. ie., + // requesting again extends the search. ie., // if a peer we requested from is gone we issue a new request, so the number of active // requests never decreases for { @@ -209,13 +215,13 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { // search timeout: too much time passed since the last request, // extend the search to a new peer if we can find one case <-waitC: - log.Trace("search timed out: rerequesting", "request addr", f.addr) + log.Trace("search timed out: requesting", "request addr", f.addr) doRequest = requested // all Fetcher context closed, can quit case <-ctx.Done(): log.Trace("terminate fetcher", "request addr", f.addr) - // TODO: send cancelations to all peers left over in peers map (i.e., those we requested from) + // TODO: send cancellations to all peers left over in peers map (i.e., those we requested from) return } @@ -231,7 +237,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { // if wait channel is not set, set it to a timer if requested { if wait == nil { - wait = time.NewTimer(searchTimeout) + wait = time.NewTimer(f.searchTimeout) defer wait.Stop() waitC = wait.C } else { @@ -242,8 +248,8 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { default: } } - // reset the timer to go off after searchTimeout - wait.Reset(searchTimeout) + // reset the timer to go off after defaultSearchTimeout + wait.Reset(f.searchTimeout) } } doRequest = false diff --git a/swarm/network/fetcher_test.go b/swarm/network/fetcher_test.go index 3a926f4758..563c6cece7 100644 --- a/swarm/network/fetcher_test.go +++ b/swarm/network/fetcher_test.go @@ -284,15 +284,11 @@ func TestFetcherRetryOnTimeout(t *testing.T) { requester := newMockRequester() addr := make([]byte, 32) fetcher := NewFetcher(addr, requester.doRequest, true) + // set searchTimeOut to low value so the test is quicker + fetcher.searchTimeout = 250 * time.Millisecond peersToSkip := &sync.Map{} - // set searchTimeOut to low value so the test is quicker - defer func(t time.Duration) { - searchTimeout = t - }(searchTimeout) - searchTimeout = 250 * time.Millisecond - ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -359,11 +355,9 @@ func TestFetcherRequestQuitRetriesRequest(t *testing.T) { addr := make([]byte, 32) fetcher := NewFetcher(addr, requester.doRequest, true) - // make sure searchTimeout is long so it is sure the request is not retried because of timeout - defer func(t time.Duration) { - searchTimeout = t - }(searchTimeout) - searchTimeout = 10 * time.Second + // make sure the searchTimeout is long so it is sure the request is not + // retried because of timeout + fetcher.searchTimeout = 10 * time.Second peersToSkip := &sync.Map{} diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 7d52f26f79..9f4245603f 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/pot" + sv "github.com/ethereum/go-ethereum/swarm/version" ) /* @@ -168,82 +169,115 @@ func (k *Kademlia) Register(peers ...*BzzAddr) error { return nil } -// SuggestPeer returns a known peer for the lowest proximity bin for the -// lowest bincount below depth -// naturally if there is an empty row it returns a peer for that -func (k *Kademlia) SuggestPeer() (a *BzzAddr, o int, want bool) { +// SuggestPeer returns an unconnected peer address as a peer suggestion for connection +func (k *Kademlia) SuggestPeer() (suggestedPeer *BzzAddr, saturationDepth int, changed bool) { k.lock.Lock() defer k.lock.Unlock() - minsize := k.MinBinSize - depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) - // if there is a callable neighbour within the current proxBin, connect - // this makes sure nearest neighbour set is fully connected - var ppo int - k.addrs.EachNeighbour(k.base, Pof, func(val pot.Val, po int) bool { - if po < depth { - return false - } - e := val.(*entry) - c := k.callable(e) - if c { - a = e.BzzAddr - } - ppo = po - return !c - }) - if a != nil { - log.Trace(fmt.Sprintf("%08x candidate nearest neighbour found: %v (%v)", k.BaseAddr()[:4], a, ppo)) - return a, 0, false - } - - var bpo []int - prev := -1 + radius := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) + // collect undersaturated bins in ascending order of number of connected peers + // and from shallow to deep (ascending order of PO) + // insert them in a map of bin arrays, keyed with the number of connected peers + saturation := make(map[int][]int) + var lastPO int // the last non-empty PO bin in the iteration + saturationDepth = -1 // the deepest PO such that all shallower bins have >= k.MinBinSize peers + var pastDepth bool // whether po of iteration >= depth k.conns.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val) bool) bool) bool { - prev++ - for ; prev < po; prev++ { - bpo = append(bpo, prev) - minsize = 0 + // process skipped empty bins + for ; lastPO < po; lastPO++ { + // find the lowest unsaturated bin + if saturationDepth == -1 { + saturationDepth = lastPO + } + // if there is an empty bin, depth is surely passed + pastDepth = true + saturation[0] = append(saturation[0], lastPO) } - if size < minsize { - bpo = append(bpo, po) - minsize = size + lastPO = po + 1 + // past radius, depth is surely passed + if po >= radius { + pastDepth = true } - return size > 0 && po < depth + // beyond depth the bin is treated as unsaturated even if size >= k.MinBinSize + // in order to achieve full connectivity to all neighbours + if pastDepth && size >= k.MinBinSize { + size = k.MinBinSize - 1 + } + // process non-empty unsaturated bins + if size < k.MinBinSize { + // find the lowest unsaturated bin + if saturationDepth == -1 { + saturationDepth = po + } + saturation[size] = append(saturation[size], po) + } + return true }) - // all buckets are full, ie., minsize == k.MinBinSize - if len(bpo) == 0 { + // to trigger peer requests for peers closer than closest connection, include + // all bins from nearest connection upto nearest address as unsaturated + var nearestAddrAt int + k.addrs.EachNeighbour(k.base, Pof, func(_ pot.Val, po int) bool { + nearestAddrAt = po + return false + }) + // including bins as size 0 has the effect that requesting connection + // is prioritised over non-empty shallower bins + for ; lastPO <= nearestAddrAt; lastPO++ { + saturation[0] = append(saturation[0], lastPO) + } + // all PO bins are saturated, ie., minsize >= k.MinBinSize, no peer suggested + if len(saturation) == 0 { return nil, 0, false } - // as long as we got candidate peers to connect to - // dont ask for new peers (want = false) - // try to select a candidate peer - // find the first callable peer - nxt := bpo[0] - k.addrs.EachBin(k.base, Pof, nxt, func(po, _ int, f func(func(pot.Val) bool) bool) bool { - // for each bin (up until depth) we find callable candidate peers - if po >= depth { - return false + // find the first callable peer in the address book + // starting from the bins with smallest size proceeding from shallow to deep + // for each bin (up until neighbourhood radius) we find callable candidate peers + for size := 0; size < k.MinBinSize && suggestedPeer == nil; size++ { + bins, ok := saturation[size] + if !ok { + // no bin with this size + continue } - return f(func(val pot.Val) bool { - e := val.(*entry) - c := k.callable(e) - if c { - a = e.BzzAddr + cur := 0 + curPO := bins[0] + k.addrs.EachBin(k.base, Pof, curPO, func(po, _ int, f func(func(pot.Val) bool) bool) bool { + curPO = bins[cur] + // find the next bin that has size size + if curPO == po { + cur++ + } else { + // skip bins that have no addresses + for ; cur < len(bins) && curPO < po; cur++ { + curPO = bins[cur] + } + if po < curPO { + cur-- + return true + } + // stop if there are no addresses + if curPO < po { + return false + } } - return !c + // curPO found + // find a callable peer out of the addresses in the unsaturated bin + // stop if found + f(func(val pot.Val) bool { + e := val.(*entry) + if k.callable(e) { + suggestedPeer = e.BzzAddr + return false + } + return true + }) + return cur < len(bins) && suggestedPeer == nil }) - }) - // found a candidate - if a != nil { - return a, 0, false } - // no candidate peer found, request for the short bin - var changed bool - if uint8(nxt) < k.depth { - k.depth = uint8(nxt) - changed = true + + if uint8(saturationDepth) < k.depth { + k.depth = uint8(saturationDepth) + return suggestedPeer, saturationDepth, true } - return a, nxt, changed + return suggestedPeer, 0, false } // On inserts the peer as a kademlia peer into the live peers @@ -398,29 +432,25 @@ func (k *Kademlia) eachAddr(base []byte, o int, f func(*BzzAddr, int) bool) { }) } +// NeighbourhoodDepth returns the depth for the pot, see depthForPot func (k *Kademlia) NeighbourhoodDepth() (depth int) { k.lock.RLock() defer k.lock.RUnlock() return depthForPot(k.conns, k.NeighbourhoodSize, k.base) } -// depthForPot returns the proximity order that defines the distance of -// the nearest neighbour set with cardinality >= NeighbourhoodSize -// if there is altogether less than NeighbourhoodSize peers it returns 0 +// neighbourhoodRadiusForPot returns the neighbourhood radius of the kademlia +// neighbourhood radius encloses the nearest neighbour set with size >= neighbourhoodSize +// i.e., neighbourhood radius is the deepest PO such that all bins not shallower altogether +// contain at least neighbourhoodSize connected peers +// if there is altogether less than neighbourhoodSize peers connected, it returns 0 // caller must hold the lock -func depthForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int) { +func neighbourhoodRadiusForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int) { if p.Size() <= neighbourhoodSize { return 0 } - // total number of peers in iteration var size int - - // determining the depth is a two-step process - // first we find the proximity bin of the shallowest of the NeighbourhoodSize peers - // the numeric value of depth cannot be higher than this - var maxDepth int - f := func(v pot.Val, i int) bool { // po == 256 means that addr is the pivot address(self) if i == 256 { @@ -431,13 +461,30 @@ func depthForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int // this means we have all nn-peers. // depth is by default set to the bin of the farthest nn-peer if size == neighbourhoodSize { - maxDepth = i + depth = i return false } return true } p.EachNeighbour(pivotAddr, Pof, f) + return depth +} + +// depthForPot returns the depth for the pot +// depth is the radius of the minimal extension of nearest neighbourhood that +// includes all empty PO bins. I.e., depth is the deepest PO such that +// - it is not deeper than neighbourhood radius +// - all bins shallower than depth are not empty +// caller must hold the lock +func depthForPot(p *pot.Pot, neighbourhoodSize int, pivotAddr []byte) (depth int) { + if p.Size() <= neighbourhoodSize { + return 0 + } + // determining the depth is a two-step process + // first we find the proximity bin of the shallowest of the neighbourhoodSize peers + // the numeric value of depth cannot be higher than this + maxDepth := neighbourhoodRadiusForPot(p, neighbourhoodSize, pivotAddr) // the second step is to test for empty bins in order from shallowest to deepest // if an empty bin is found, this will be the actual depth @@ -506,6 +553,9 @@ func (k *Kademlia) string() string { var rows []string rows = append(rows, "=========================================================================") + if len(sv.GitCommit) > 0 { + rows = append(rows, fmt.Sprintf("commit hash: %s", sv.GitCommit)) + } rows = append(rows, fmt.Sprintf("%v KΛÐΞMLIΛ hive: queen's address: %x", time.Now().UTC().Format(time.UnixDate), k.BaseAddr()[:3])) rows = append(rows, fmt.Sprintf("population: %d (%d), NeighbourhoodSize: %d, MinBinSize: %d, MaxBinSize: %d", k.conns.Size(), k.addrs.Size(), k.NeighbourhoodSize, k.MinBinSize, k.MaxBinSize)) @@ -627,21 +677,20 @@ func NewPeerPotMap(neighbourhoodSize int, addrs [][]byte) map[string]*PeerPot { return ppmap } -// saturation iterates through all peers and -// returns the smallest po value in which the node has less than n peers -// if the iterator reaches depth, then value for depth is returned -// TODO move to separate testing tools file -// TODO this function will stop at the first bin with less than MinBinSize peers, even if there are empty bins between that bin and the depth. This may not be correct behavior +// saturation returns the smallest po value in which the node has less than MinBinSize peers +// if the iterator reaches neighbourhood radius, then the last bin + 1 is returned func (k *Kademlia) saturation() int { prev := -1 - k.addrs.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val) bool) bool) bool { + radius := neighbourhoodRadiusForPot(k.conns, k.NeighbourhoodSize, k.base) + k.conns.EachBin(k.base, Pof, 0, func(po, size int, f func(func(val pot.Val) bool) bool) bool { prev++ + if po >= radius { + return false + } return prev == po && size >= k.MinBinSize }) - // TODO evaluate whether this check cannot just as well be done within the eachbin - depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) - if depth < prev { - return depth + if prev < 0 { + return 0 } return prev } @@ -743,6 +792,9 @@ type Health struct { func (k *Kademlia) Healthy(pp *PeerPot) *Health { k.lock.RLock() defer k.lock.RUnlock() + if len(pp.NNSet) < k.NeighbourhoodSize { + log.Warn("peerpot NNSet < NeighbourhoodSize") + } gotnn, countgotnn, culpritsgotnn := k.connectedNeighbours(pp.NNSet) knownn, countknownn, culpritsknownn := k.knowNeighbours(pp.NNSet) depth := depthForPot(k.conns, k.NeighbourhoodSize, k.base) diff --git a/swarm/network/kademlia_test.go b/swarm/network/kademlia_test.go index fcb277fde7..8a724756b6 100644 --- a/swarm/network/kademlia_test.go +++ b/swarm/network/kademlia_test.go @@ -17,7 +17,6 @@ package network import ( - "bytes" "fmt" "os" "testing" @@ -43,39 +42,46 @@ func testKadPeerAddr(s string) *BzzAddr { func newTestKademliaParams() *KadParams { params := NewKadParams() - // TODO why is this 1? - params.MinBinSize = 1 + params.MinBinSize = 2 params.NeighbourhoodSize = 2 return params } -func newTestKademlia(b string) *Kademlia { +type testKademlia struct { + *Kademlia + t *testing.T +} + +func newTestKademlia(t *testing.T, b string) *testKademlia { base := pot.NewAddressFromString(b) - return NewKademlia(base, newTestKademliaParams()) + return &testKademlia{ + Kademlia: NewKademlia(base, newTestKademliaParams()), + t: t, + } } -func newTestKadPeer(k *Kademlia, s string, lightNode bool) *Peer { - return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, k) +func (tk *testKademlia) newTestKadPeer(s string, lightNode bool) *Peer { + return NewPeer(&BzzPeer{BzzAddr: testKadPeerAddr(s), LightNode: lightNode}, tk.Kademlia) } -func On(k *Kademlia, ons ...string) { +func (tk *testKademlia) On(ons ...string) { for _, s := range ons { - k.On(newTestKadPeer(k, s, false)) + tk.Kademlia.On(tk.newTestKadPeer(s, false)) } } -func Off(k *Kademlia, offs ...string) { +func (tk *testKademlia) Off(offs ...string) { for _, s := range offs { - k.Off(newTestKadPeer(k, s, false)) + tk.Kademlia.Off(tk.newTestKadPeer(s, false)) } } -func Register(k *Kademlia, regs ...string) { +func (tk *testKademlia) Register(regs ...string) { var as []*BzzAddr for _, s := range regs { as = append(as, testKadPeerAddr(s)) } - err := k.Register(as...) + err := tk.Kademlia.Register(as...) if err != nil { panic(err.Error()) } @@ -169,76 +175,71 @@ func TestHealthStrict(t *testing.T) { // base address is all zeros // no peers // unhealthy (and lonely) - k := newTestKademlia("11111111") - assertHealth(t, k, false, false) + tk := newTestKademlia(t, "11111111") + tk.checkHealth(false, false) // know one peer but not connected // unhealthy - Register(k, "11100000") - log.Trace(k.String()) - assertHealth(t, k, false, false) + tk.Register("11100000") + tk.checkHealth(false, false) // know one peer and connected // healthy - On(k, "11100000") - assertHealth(t, k, true, false) + tk.On("11100000") + tk.checkHealth(true, false) // know two peers, only one connected // unhealthy - Register(k, "11111100") - log.Trace(k.String()) - assertHealth(t, k, false, false) + tk.Register("11111100") + tk.checkHealth(false, false) // know two peers and connected to both // healthy - On(k, "11111100") - assertHealth(t, k, true, false) + tk.On("11111100") + tk.checkHealth(true, false) // know three peers, connected to the two deepest // healthy - Register(k, "00000000") - log.Trace(k.String()) - assertHealth(t, k, true, false) + tk.Register("00000000") + tk.checkHealth(true, false) // know three peers, connected to all three // healthy - On(k, "00000000") - assertHealth(t, k, true, false) + tk.On("00000000") + tk.checkHealth(true, false) // add fourth peer deeper than current depth // unhealthy - Register(k, "11110000") - log.Trace(k.String()) - assertHealth(t, k, false, false) + tk.Register("11110000") + tk.checkHealth(false, false) // connected to three deepest peers // healthy - On(k, "11110000") - assertHealth(t, k, true, false) + tk.On("11110000") + tk.checkHealth(true, false) // add additional peer in same bin as deepest peer // unhealthy - Register(k, "11111101") - log.Trace(k.String()) - assertHealth(t, k, false, false) + tk.Register("11111101") + tk.checkHealth(false, false) // four deepest of five peers connected // healthy - On(k, "11111101") - assertHealth(t, k, true, false) + tk.On("11111101") + tk.checkHealth(true, false) } -func assertHealth(t *testing.T, k *Kademlia, expectHealthy bool, expectSaturation bool) { - t.Helper() - kid := common.Bytes2Hex(k.BaseAddr()) - addrs := [][]byte{k.BaseAddr()} - k.EachAddr(nil, 255, func(addr *BzzAddr, po int) bool { +func (tk *testKademlia) checkHealth(expectHealthy bool, expectSaturation bool) { + tk.t.Helper() + kid := common.Bytes2Hex(tk.BaseAddr()) + addrs := [][]byte{tk.BaseAddr()} + tk.EachAddr(nil, 255, func(addr *BzzAddr, po int) bool { addrs = append(addrs, addr.Address()) return true }) - pp := NewPeerPotMap(k.NeighbourhoodSize, addrs) - healthParams := k.Healthy(pp[kid]) + pp := NewPeerPotMap(tk.NeighbourhoodSize, addrs) + healthParams := tk.Healthy(pp[kid]) // definition of health, all conditions but be true: // - we at least know one peer @@ -246,23 +247,23 @@ func assertHealth(t *testing.T, k *Kademlia, expectHealthy bool, expectSaturatio // - we are connected to all known neighbors health := healthParams.KnowNN && healthParams.ConnectNN && healthParams.CountKnowNN > 0 if expectHealthy != health { - t.Fatalf("expected kademlia health %v, is %v\n%v", expectHealthy, health, k.String()) + tk.t.Fatalf("expected kademlia health %v, is %v\n%v", expectHealthy, health, tk.String()) } } -func testSuggestPeer(k *Kademlia, expAddr string, expPo int, expWant bool) error { - addr, o, want := k.SuggestPeer() - log.Trace("suggestpeer return", "a", addr, "o", o, "want", want) +func (tk *testKademlia) checkSuggestPeer(expAddr string, expDepth int, expChanged bool) { + tk.t.Helper() + addr, depth, changed := tk.SuggestPeer() + log.Trace("suggestPeer return", "addr", addr, "depth", depth, "changed", changed) if binStr(addr) != expAddr { - return fmt.Errorf("incorrect peer address suggested. expected %v, got %v", expAddr, binStr(addr)) + tk.t.Fatalf("incorrect peer address suggested. expected %v, got %v", expAddr, binStr(addr)) } - if o != expPo { - return fmt.Errorf("incorrect prox order suggested. expected %v, got %v", expPo, o) + if depth != expDepth { + tk.t.Fatalf("incorrect saturation depth suggested. expected %v, got %v", expDepth, depth) } - if want != expWant { - return fmt.Errorf("expected SuggestPeer to want peers: %v", expWant) + if changed != expChanged { + tk.t.Fatalf("expected depth change = %v, got %v", expChanged, changed) } - return nil } func binStr(a *BzzAddr) string { @@ -272,609 +273,184 @@ func binStr(a *BzzAddr) string { return pot.ToBin(a.Address())[:8] } -// TODO explain why this bug occurred and how it should have been mitigated -func TestSuggestPeerBug(t *testing.T) { - // 2 row gap, unsaturated proxbin, no callables -> want PO 0 - k := newTestKademlia("00000000") - On(k, - "10000000", "11000000", - "01000000", - - "00010000", "00011000", - ) - Off(k, - "01000000", - ) - err := testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatal(err.Error()) - } -} - func TestSuggestPeerFindPeers(t *testing.T) { - t.Skip("The SuggestPeers implementation seems to have weaknesses exposed by the change in the new depth calculation. The results are no longer predictable") + tk := newTestKademlia(t, "00000000") + tk.On("00100000") + tk.checkSuggestPeer("", 0, false) - testnum := 0 - // test 0 - // 2 row gap, unsaturated proxbin, no callables -> want PO 0 - k := newTestKademlia("00000000") - On(k, "00100000") - err := testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("00010000") + tk.checkSuggestPeer("", 0, false) - // test 1 - // 2 row gap, saturated proxbin, no callables -> want PO 0 - On(k, "00010000") - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("10000000", "10000001") + tk.checkSuggestPeer("", 0, false) - // test 2 - // 1 row gap (1 less), saturated proxbin, no callables -> want PO 1 - On(k, "10000000") - err = testSuggestPeer(k, "", 1, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000000") + tk.Off("10000001") + tk.checkSuggestPeer("10000001", 0, true) - // test 3 - // no gap (1 less), saturated proxbin, no callables -> do not want more - On(k, "01000000", "00100001") - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("00100001") + tk.Off("01000000") + tk.checkSuggestPeer("01000000", 0, false) - // test 4 - // oversaturated proxbin, > do not want more - On(k, "00100001") - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ - - // test 5 - // reintroduce gap, disconnected peer callable - Off(k, "01000000") - log.Trace(k.String()) - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ - - // test 6 // second time disconnected peer not callable // with reasonably set Interval - log.Trace("foo") - log.Trace(k.String()) - err = testSuggestPeer(k, "", 1, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.checkSuggestPeer("", 0, false) - // test 6 // on and off again, peer callable again - On(k, "01000000") - Off(k, "01000000") - log.Trace(k.String()) - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000000") + tk.Off("01000000") + tk.checkSuggestPeer("01000000", 0, false) - // test 7 - // new closer peer appears, it is immediately wanted - On(k, "01000000") - Register(k, "00010001") - err = testSuggestPeer(k, "00010001", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000000", "10000001") + tk.checkSuggestPeer("", 0, false) - // test 8 - // PO1 disconnects - On(k, "00010001") - log.Info(k.String()) - Off(k, "01000000") - log.Info(k.String()) - // second time, gap filling - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Register("00010001") + tk.checkSuggestPeer("00010001", 0, false) - // test 9 - On(k, "01000000") - log.Info(k.String()) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("00010001") + tk.Off("01000000") + tk.checkSuggestPeer("01000000", 0, false) - // test 10 - k.MinBinSize = 2 - log.Info(k.String()) - err = testSuggestPeer(k, "", 0, true) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000000") + tk.checkSuggestPeer("", 0, false) - // test 11 - Register(k, "01000001") - log.Info(k.String()) - err = testSuggestPeer(k, "01000001", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Register("01000001") + tk.checkSuggestPeer("01000001", 0, false) - // test 12 - On(k, "10000001") - log.Trace(fmt.Sprintf("Kad:\n%v", k.String())) - err = testSuggestPeer(k, "", 1, true) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000001") + tk.checkSuggestPeer("", 0, false) - // test 13 - On(k, "01000001") - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Register("10000010", "01000010", "00100010") + tk.checkSuggestPeer("", 0, false) - // test 14 - k.MinBinSize = 3 - Register(k, "10000010") - err = testSuggestPeer(k, "10000010", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Register("00010010") + tk.checkSuggestPeer("00010010", 0, false) - // test 15 - On(k, "10000010") - err = testSuggestPeer(k, "", 1, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Off("00100001") + tk.checkSuggestPeer("00100010", 2, true) - // test 16 - On(k, "01000010") - err = testSuggestPeer(k, "", 2, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.Off("01000001") + tk.checkSuggestPeer("01000010", 1, true) - // test 17 - On(k, "00100010") - err = testSuggestPeer(k, "", 3, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.checkSuggestPeer("01000001", 0, false) + tk.checkSuggestPeer("00100001", 0, false) + tk.checkSuggestPeer("", 0, false) - // test 18 - On(k, "00010010") - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatalf("%d %v", testnum, err.Error()) - } - testnum++ + tk.On("01000001", "00100001") + tk.Register("10000100", "01000100", "00100100") + tk.Register("00000100", "00000101", "00000110") + tk.Register("00000010", "00000011", "00000001") + + tk.checkSuggestPeer("00000110", 0, false) + tk.checkSuggestPeer("00000101", 0, false) + tk.checkSuggestPeer("00000100", 0, false) + tk.checkSuggestPeer("00000011", 0, false) + tk.checkSuggestPeer("00000010", 0, false) + tk.checkSuggestPeer("00000001", 0, false) + tk.checkSuggestPeer("", 0, false) } // a node should stay in the address book if it's removed from the kademlia func TestOffEffectingAddressBookNormalNode(t *testing.T) { - k := newTestKademlia("00000000") + tk := newTestKademlia(t, "00000000") // peer added to kademlia - k.On(newTestKadPeer(k, "01000000", false)) + tk.On("01000000") // peer should be in the address book - if k.addrs.Size() != 1 { + if tk.addrs.Size() != 1 { t.Fatal("known peer addresses should contain 1 entry") } // peer should be among live connections - if k.conns.Size() != 1 { + if tk.conns.Size() != 1 { t.Fatal("live peers should contain 1 entry") } // remove peer from kademlia - k.Off(newTestKadPeer(k, "01000000", false)) + tk.Off("01000000") // peer should be in the address book - if k.addrs.Size() != 1 { + if tk.addrs.Size() != 1 { t.Fatal("known peer addresses should contain 1 entry") } // peer should not be among live connections - if k.conns.Size() != 0 { + if tk.conns.Size() != 0 { t.Fatal("live peers should contain 0 entry") } } // a light node should not be in the address book func TestOffEffectingAddressBookLightNode(t *testing.T) { - k := newTestKademlia("00000000") + tk := newTestKademlia(t, "00000000") // light node peer added to kademlia - k.On(newTestKadPeer(k, "01000000", true)) + tk.Kademlia.On(tk.newTestKadPeer("01000000", true)) // peer should not be in the address book - if k.addrs.Size() != 0 { + if tk.addrs.Size() != 0 { t.Fatal("known peer addresses should contain 0 entry") } // peer should be among live connections - if k.conns.Size() != 1 { + if tk.conns.Size() != 1 { t.Fatal("live peers should contain 1 entry") } // remove peer from kademlia - k.Off(newTestKadPeer(k, "01000000", true)) + tk.Kademlia.Off(tk.newTestKadPeer("01000000", true)) // peer should not be in the address book - if k.addrs.Size() != 0 { + if tk.addrs.Size() != 0 { t.Fatal("known peer addresses should contain 0 entry") } // peer should not be among live connections - if k.conns.Size() != 0 { + if tk.conns.Size() != 0 { t.Fatal("live peers should contain 0 entry") } } func TestSuggestPeerRetries(t *testing.T) { - k := newTestKademlia("00000000") - k.RetryInterval = int64(300 * time.Millisecond) // cycle - k.MaxRetries = 50 - k.RetryExponent = 2 + tk := newTestKademlia(t, "00000000") + tk.RetryInterval = int64(300 * time.Millisecond) // cycle + tk.MaxRetries = 50 + tk.RetryExponent = 2 sleep := func(n int) { - ts := k.RetryInterval + ts := tk.RetryInterval for i := 1; i < n; i++ { - ts *= int64(k.RetryExponent) + ts *= int64(tk.RetryExponent) } time.Sleep(time.Duration(ts)) } - Register(k, "01000000") - On(k, "00000001", "00000010") - err := testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.Register("01000000") + tk.On("00000001", "00000010") + tk.checkSuggestPeer("01000000", 0, false) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("", 0, false) sleep(1) - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("01000000", 0, false) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("", 0, false) sleep(1) - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("01000000", 0, false) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("", 0, false) sleep(2) - err = testSuggestPeer(k, "01000000", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("01000000", 0, false) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatal(err.Error()) - } + tk.checkSuggestPeer("", 0, false) sleep(2) - err = testSuggestPeer(k, "", 0, false) - if err != nil { - t.Fatal(err.Error()) - } - + tk.checkSuggestPeer("", 0, false) } func TestKademliaHiveString(t *testing.T) { - k := newTestKademlia("00000000") - On(k, "01000000", "00100000") - Register(k, "10000000", "10000001") - k.MaxProxDisplay = 8 - h := k.String() - expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), NeighbourhoodSize: 2, MinBinSize: 1, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000 0 | 2 8100 (0) 8000 (0)\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" + tk := newTestKademlia(t, "00000000") + tk.On("01000000", "00100000") + tk.Register("10000000", "10000001") + tk.MaxProxDisplay = 8 + h := tk.String() + expH := "\n=========================================================================\nMon Feb 27 12:10:28 UTC 2017 KΛÐΞMLIΛ hive: queen's address: 000000\npopulation: 2 (4), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000 0 | 2 8100 (0) 8000 (0)\n001 1 4000 | 1 4000 (0)\n002 1 2000 | 1 2000 (0)\n003 0 | 0\n004 0 | 0\n005 0 | 0\n006 0 | 0\n007 0 | 0\n=========================================================================" if expH[104:] != h[104:] { t.Fatalf("incorrect hive output. expected %v, got %v", expH, h) } } -// testKademliaCase constructs the kademlia and PeerPot map to validate -// the SuggestPeer and Healthy methods for provided hex-encoded addresses. -// Argument pivotAddr is the address of the kademlia. -func testKademliaCase(t *testing.T, pivotAddr string, addrs ...string) { - - t.Skip("this test relies on SuggestPeer which is now not reliable. See description in TestSuggestPeerFindPeers") - addr := common.Hex2Bytes(pivotAddr) - var byteAddrs [][]byte - for _, ahex := range addrs { - byteAddrs = append(byteAddrs, common.Hex2Bytes(ahex)) - } - - k := NewKademlia(addr, NewKadParams()) - - // our pivot kademlia is the last one in the array - for _, a := range byteAddrs { - if bytes.Equal(a, addr) { - continue - } - p := &BzzAddr{OAddr: a, UAddr: a} - if err := k.Register(p); err != nil { - t.Fatalf("a %x addr %x: %v", a, addr, err) - } - } - - ppmap := NewPeerPotMap(k.NeighbourhoodSize, byteAddrs) - - pp := ppmap[pivotAddr] - - for { - a, _, _ := k.SuggestPeer() - if a == nil { - break - } - k.On(NewPeer(&BzzPeer{BzzAddr: a}, k)) - } - - h := k.Healthy(pp) - if !(h.ConnectNN && h.KnowNN && h.CountKnowNN > 0) { - t.Fatalf("not healthy: %#v\n%v", h, k.String()) - } -} - -/* -The regression test for the following invalid kademlia edge case. - -Addresses used in this test are discovered as part of the simulation network -in higher level tests for streaming. They were generated randomly. - -========================================================================= -Mon Apr 9 12:18:24 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 7efef1 -population: 9 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 -000 2 d7e5 ec56 | 18 ec56 (0) d7e5 (0) d9e0 (0) c735 (0) -001 2 18f1 3176 | 14 18f1 (0) 10bb (0) 10d1 (0) 0421 (0) -002 2 52aa 47cd | 11 52aa (0) 51d9 (0) 5161 (0) 5130 (0) -003 1 646e | 1 646e (0) -004 0 | 3 769c (0) 76d1 (0) 7656 (0) -============ DEPTH: 5 ========================================== -005 1 7a48 | 1 7a48 (0) -006 1 7cbd | 1 7cbd (0) -007 0 | 0 -008 0 | 0 -009 0 | 0 -010 0 | 0 -011 0 | 0 -012 0 | 0 -013 0 | 0 -014 0 | 0 -015 0 | 0 -========================================================================= -*/ -func TestKademliaCase1(t *testing.T) { - testKademliaCase(t, - "7efef1c41d77f843ad167be95f6660567eb8a4a59f39240000cce2e0d65baf8e", - "ec560e6a4806aa37f147ee83687f3cf044d9953e61eedb8c34b6d50d9e2c5623", - "646e9540c84f6a2f9cf6585d45a4c219573b4fd1b64a3c9a1386fc5cf98c0d4d", - "18f13c5fba653781019025ab10e8d2fdc916d6448729268afe9e928ffcdbb8e8", - "317617acf99b4ffddda8a736f8fc6c6ede0bf690bc23d834123823e6d03e2f69", - "d7e52d9647a5d1c27a68c3ee65d543be3947ae4b68537b236d71ef9cb15fb9ab", - "7a48f75f8ca60487ae42d6f92b785581b40b91f2da551ae73d5eae46640e02e8", - "7cbd42350bde8e18ae5b955b5450f8e2cef3419f92fbf5598160c60fd78619f0", - "52aa3ddec61f4d48dd505a2385403c634f6ad06ee1d99c5c90a5ba6006f9af9c", - "47cdb6fa93eeb8bc91a417ff4e3b14a9c2ea85137462e2f575fae97f0c4be60d", - "5161943eb42e2a03e715fe8afa1009ff5200060c870ead6ab103f63f26cb107f", - "a38eaa1255f76bf883ca0830c86e8c4bb7eed259a8348aae9b03f21f90105bee", - "b2522bdf1ab26f324e75424fdf6e493b47e8a27687fe76347607b344fc010075", - "5bd7213964efb2580b91d02ac31ef126838abeba342f5dbdbe8d4d03562671a2", - "0b531adb82744768b694d7f94f73d4f0c9de591266108daeb8c74066bfc9c9ca", - "28501f59f70e888d399570145ed884353e017443c675aa12731ada7c87ea14f7", - "4a45f1fc63e1a9cb9dfa44c98da2f3d20c2923e5d75ff60b2db9d1bdb0c54d51", - "b193431ee35cd32de95805e7c1c749450c47486595aae7195ea6b6019a64fd61", - "baebf36a1e35a7ed834e1c72faf44ba16c159fa47d3289ceb3ca35fefa8739b5", - "a3659bd32e05fa36c8d20dbaaed8362bf1a8a7bd116aed62d8a43a2efbdf513f", - "10d1b50881a4770ebebdd0a75589dabb931e6716747b0f65fd6b080b88c4fdb6", - "3c76b8ca5c7ce6a03320646826213f59229626bf5b9d25da0c3ec0662dcb8ff3", - "4d72a04ddeb851a68cd197ef9a92a3e2ff01fbbff638e64929dd1a9c2e150112", - "c7353d320987956075b5bc1668571c7a36c800d5598fdc4832ec6569561e15d1", - "d9e0c7c90878c20ab7639d5954756f54775404b3483407fe1b483635182734f6", - "8fca67216b7939c0824fb06c5279901a94da41da9482b000f56df9906736ee75", - "460719d7f7aa7d7438f0eaf30333484fa3bd0f233632c10ba89e6e46dd3604be", - "0421d92c8a1c79ed5d01305a3d25aaf22a8f5f9e3d4bc80da47ee16ce20465fe", - "3441d9d9c0f05820a1bb6459fc7d8ef266a1bd929e7db939a10f544efe8261ea", - "ab198a66c293586746758468c610e5d3914d4ce629147eff6dd55a31f863ff8f", - "3a1c8c16b0763f3d2c35269f454ff779d1255e954d2deaf6c040fb3f0bcdc945", - "5561c0ea3b203e173b11e6aa9d0e621a4e10b1d8b178b8fe375220806557b823", - "7656caccdc79cd8d7ce66d415cc96a718e8271c62fb35746bfc2b49faf3eebf3", - "5130594fd54c1652cf2debde2c4204573ed76555d1e26757fe345b409af1544a", - "76d1e83c71ca246d042e37ff1db181f2776265fbcfdc890ce230bfa617c9c2f0", - "89580231962624c53968c1b0095b4a2732b2a2640a19fdd7d21fd064fcc0a5ef", - "3d10d001fff44680c7417dd66ecf2e984f0baa20a9bbcea348583ba5ff210c4f", - "43754e323f0f3a1155b1852bd6edd55da86b8c4cfe3df8b33733fca50fc202b8", - "a9e7b1bb763ae6452ddcacd174993f82977d81a85206bb2ae3c842e2d8e19b4c", - "10bb07da7bc7c7757f74149eff167d528a94a253cdc694a863f4d50054c00b6d", - "28f0bc1b44658548d6e05dd16d4c2fe77f1da5d48b6774bc4263b045725d0c19", - "835fbbf1d16ba7347b6e2fc552d6e982148d29c624ea20383850df3c810fa8fc", - "8e236c56a77d7f46e41e80f7092b1a68cd8e92f6156365f41813ad1ca2c6b6f3", - "51d9c857e9238c49186e37b4eccf17a82de3d5739f026f6043798ab531456e73", - "bbddf7db6a682225301f36a9fd5b0d0121d2951753e1681295f3465352ad511f", - "2690a910c33ee37b91eb6c4e0731d1d345e2dc3b46d308503a6e85bbc242c69e", - "769ce86aa90b518b7ed382f9fdacfbed93574e18dc98fe6c342e4f9f409c2d5a", - "ba3bebec689ce51d3e12776c45f80d25164fdfb694a8122d908081aaa2e7122c", - "3a51f4146ea90a815d0d283d1ceb20b928d8b4d45875e892696986a3c0d8fb9b", - "81968a2d8fb39114342ee1da85254ec51e0608d7f0f6997c2a8354c260a71009", - ) -} - -/* -The regression test for the following invalid kademlia edge case. - -Addresses used in this test are discovered as part of the simulation network -in higher level tests for streaming. They were generated randomly. - -========================================================================= -Mon Apr 9 18:43:48 UTC 2018 KΛÐΞMLIΛ hive: queen's address: bc7f3b -population: 9 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 -000 2 0f49 67ff | 28 0f49 (0) 0211 (0) 07b2 (0) 0703 (0) -001 2 e84b f3a4 | 13 f3a4 (0) e84b (0) e58b (0) e60b (0) -002 1 8dba | 1 8dba (0) -003 2 a008 ad72 | 2 ad72 (0) a008 (0) -004 0 | 3 b61f (0) b27f (0) b027 (0) -============ DEPTH: 5 ========================================== -005 1 ba19 | 1 ba19 (0) -006 0 | 0 -007 1 bdd6 | 1 bdd6 (0) -008 0 | 0 -009 0 | 0 -010 0 | 0 -011 0 | 0 -012 0 | 0 -013 0 | 0 -014 0 | 0 -015 0 | 0 -========================================================================= -*/ -func TestKademliaCase2(t *testing.T) { - testKademliaCase(t, - "bc7f3b6a4a7e3c91b100ca6680b6c06ff407972b88956324ca853295893e0237", "67ffb61d3aa27449d277016188f35f19e2321fbda5008c68cf6303faa080534f", "600cd54c842eadac1729c04abfc369bc244572ca76117105b9dd910283b82730", "d955a05409650de151218557425105a8aa2867bb6a0e0462fa1cf90abcf87ad6", "7a6b726de45abdf7bb3e5fd9fb0dc8932270ca4dedef92238c80c05bcdb570e3", "263e99424ebfdb652adb4e3dcd27d59e11bb7ae1c057b3ef6f390d0228006254", "ba195d1a53aafde68e661c64d39db8c2a73505bf336125c15c3560de3b48b7ed", "3458c762169937115f67cabc35a6c384ed70293a8aec37b077a6c1b8e02d510e", "4ef4dc2e28ac6efdba57e134ac24dd4e0be68b9d54f7006515eb9509105f700c", "2a8782b79b0c24b9714dfd2c8ff1932bebc08aa6520b4eaeaa59ff781238890c", "625d02e960506f4524e9cdeac85b33faf3ea437fceadbd478b62b78720cf24fc", "e051a36a8c8637f520ba259c9ed3fadaf740dadc6a04c3f0e21778ebd4cd6ac4", "e34bc014fa2504f707bb3d904872b56c2fa250bee3cb19a147a0418541f1bd90", "28036dc79add95799916893890add5d8972f3b95325a509d6ded3d448f4dc652", "1b013c407794fa2e4c955d8f51cbc6bd78588a174b6548246b291281304b5409", "34f71b68698e1534095ff23ee9c35bf64c7f12b8463e7c6f6b19c25cf03928b4", "c712c6e9bbb7076832972a95890e340b94ed735935c3c0bb788e61f011b59479", "a008d5becdcda4b9dbfdaafc3cec586cf61dcf2d4b713b6168fff02e3b9f0b08", "29de15555cdbebaab214009e416ee92f947dcec5dab9894129f50f1b17138f34", "5df9449f700bd4b5a23688b68b293f2e92fa6ca524c93bc6bb9936efba9d9ada", "3ab0168a5f87fedc6a39b53c628256ac87a98670d8691bbdaaecec22418d13a2", "1ee299b2d2a74a568494130e6869e66d57982d345c482a0e0eeb285ac219ae3b", "e0e0e3b860cea9b7a74cf1b0675cc632dc64e80a02f20bbc5e96e2e8bb670606", "dc1ba6f169b0fcdcca021dcebaf39fe5d4875e7e69b854fad65687c1d7719ec0", "d321f73e42fcfb1d3a303eddf018ca5dffdcfd5567cd5ec1212f045f6a07e47d", "070320c3da7b542e5ca8aaf6a0a53d2bb5113ed264ab1db2dceee17c729edcb1", "17d314d65fdd136b50d182d2c8f5edf16e7838c2be8cf2c00abe4b406dbcd1d8", "e60b99e0a06f7d2d99d84085f67cdf8cc22a9ae22c339365d80f90289834a2b4", "02115771e18932e1f67a45f11f5bf743c5dae97fbc477d34d35c996012420eac", "3102a40eb2e5060353dd19bf61eeec8782dd1bebfcb57f4c796912252b591827", "8dbaf231062f2dc7ddaba5f9c7761b0c21292be51bf8c2ef503f31d4a2f63f79", "b02787b713c83a9f9183216310f04251994e04c2763a9024731562e8978e7cc4", "b27fe6cd33989e10909ce794c4b0b88feae286b614a59d49a3444c1a7b51ea82", "07b2d2c94fdc6fd148fe23be2ed9eff54f5e12548f29ed8416e6860fc894466f", "e58bf9f451ef62ac44ff0a9bb0610ec0fd14d423235954f0d3695e83017cbfc4", "bdd600b91bb79d1ee0053b854de308cfaa7e2abce575ea6815a0a7b3449609c2", "0f49c93c1edc7999920b21977cedd51a763940dac32e319feb9c1df2da0f3071", "7cbf0297cd41acf655cd6f960d7aaf61479edb4189d5c001cbc730861f0deb41", "79265193778d87ad626a5f59397bc075872d7302a12634ce2451a767d0a82da2", "2fe7d705f7c370b9243dbaafe007d555ff58d218822fca49d347b12a0282457c", "e84bc0c83d05e55a0080eed41dda5a795da4b9313a4da697142e69a65834cbb3", "cc4d278bd9aa0e9fb3cd8d2e0d68fb791aab5de4b120b845c409effbed47a180", "1a2317a8646cd4b6d3c4aa4cc25f676533abb689cf180787db216880a1239ad8", "cbafd6568cf8e99076208e6b6843f5808a7087897c67aad0c54694669398f889", "7b7c8357255fc37b4dae0e1af61589035fd39ff627e0938c6b3da8b4e4ec5d23", "2b8d782c1f5bac46c922cf439f6aa79f91e9ba5ffc0020d58455188a2075b334", "b61f45af2306705740742e76197a119235584ced01ef3f7cf3d4370f6c557cd1", "2775612e7cdae2780bf494c370bdcbe69c55e4a1363b1dc79ea0135e61221cce", "f3a49bb22f40885e961299abfa697a7df690a79f067bf3a4847a3ad48d826c9f", "ad724ac218dc133c0aadf4618eae21fdd0c2f3787af279846b49e2b4f97ff167", - ) -} - -/* -The regression test for the following invalid kademlia edge case. - -Addresses used in this test are discovered as part of the simulation network -in higher level tests for streaming. They were generated randomly. - -========================================================================= -Mon Apr 9 19:04:35 UTC 2018 KΛÐΞMLIΛ hive: queen's address: b4822e -population: 8 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 -000 2 786c 774b | 29 774b (0) 786c (0) 7a79 (0) 7d2f (0) -001 2 d9de cf19 | 10 cf19 (0) d9de (0) d2ff (0) d2a2 (0) -002 2 8ca1 8d74 | 5 8d74 (0) 8ca1 (0) 9793 (0) 9f51 (0) -003 0 | 0 -004 0 | 3 bfac (0) bcbb (0) bde9 (0) -005 0 | 0 -============ DEPTH: 6 ========================================== -006 1 b660 | 1 b660 (0) -007 0 | 0 -008 1 b450 | 1 b450 (0) -009 0 | 0 -010 0 | 0 -011 0 | 0 -012 0 | 0 -013 0 | 0 -014 0 | 0 -015 0 | 0 -========================================================================= -*/ -func TestKademliaCase3(t *testing.T) { - testKademliaCase(t, - "b4822e874a01b94ac3a35c821e6db131e785c2fcbb3556e84b36102caf09b091", "2ecf54ea38d58f9cfc3862e54e5854a7c506fbc640e0b38e46d7d45a19794999", "442374092be50fc7392e8dd3f6fab3158ff7f14f26ff98060aed9b2eecf0b97d", "b450a4a67fcfa3b976cf023d8f1f15052b727f712198ce901630efe2f95db191", "9a7291638eb1c989a6dd6661a42c735b23ac6605b5d3e428aa5ffe650e892c85", "67f62eeab9804cfcac02b25ebeab9113d1b9d03dd5200b1c5a324cc0163e722f", "2e4a0e4b53bca4a9d7e2734150e9f579f29a255ade18a268461b20d026c9ee90", "30dd79c5fcdaa1b106f6960c45c9fde7c046aa3d931088d98c52ab759d0b2ac4", "97936fb5a581e59753c54fa5feec493714f2218245f61f97a62eafd4699433e4", "3a2899b6e129e3e193f6e2aefb82589c948c246d2ec1d4272af32ef3b2660f44", "f0e2a8aa88e67269e9952431ef12e5b29b7f41a1871fbfc38567fad95655d607", "7fa12b3f3c5f8383bfc644b958f72a486969733fa097d8952b3eb4f7b4f73192", "360c167aad5fc992656d6010ec45fdce5bcd492ad9608bc515e2be70d4e430c1", "fe21bc969b3d8e5a64a6484a829c1e04208f26f3cd4de6afcbc172a5bd17f1f1", "b660a1f40141d7ccd282fe5bd9838744119bd1cb3780498b5173578cc5ad308f", "44dcb3370e76680e2fba8cd986ad45ff0b77ca45680ee8d950e47922c4af6226", "8ca126923d17fccb689647307b89f38aa14e2a7b9ebcf3c1e31ccf3d2291a3bc", "f0ae19ae9ce6329327cbf42baf090e084c196b0877d8c7b69997e0123be23ef8", "d2a2a217385158e3e1e348883a14bc423e57daa12077e8c49797d16121ea0810", "f5467ccd85bb4ebe768527db520a210459969a5f1fae6e07b43f519799f0b224", "68be5fd9f9d142a5099e3609011fe3bab7bb992c595999e31e0b3d1668dfb3cf", "4d49a8a476e4934afc6b5c36db9bece3ed1804f20b952da5a21b2b0de766aa73", "ea7155745ef3fb2d099513887a2ba279333ced65c65facbd890ce58bd3fce772", "cf19f51f4e848053d289ac95a9138cdd23fc3077ae913cd58cda8cc7a521b2e1", "590b1cd41c7e6144e76b5cd515a3a4d0a4317624620a3f1685f43ae68bdcd890", "d2ffe0626b5f94a7e00fa0b506e7455a3d9399c15800db108d5e715ef5f6e346", "69630878c50a91f6c2edd23a706bfa0b50bd5661672a37d67bab38e6bca3b698", "445e9067079899bb5faafaca915ae6c0f6b1b730a5a628835dd827636f7feb1e", "6461c77491f1c4825958949f23c153e6e1759a5be53abbcee17c9da3867f3141", "23a235f4083771ccc207771daceda700b525a59ab586788d4f6892e69e34a6e2", "bde99f79ef41a81607ddcf92b9f95dcbc6c3537e91e8bf740e193dc73b19485e", "177957c0e5f0fbd12b88022a91768095d193830986caec8d888097d3ff4310b8", "bcbbdbaa4cdf8352422072f332e05111b732354a35c4d7c617ce1fc3b8b42a5a", "774b6717fdfb0d1629fb9d4c04a9ca40079ae2955d7f82e897477055ed017abb", "16443bf625be6d39ecaa6f114e5d2c1d47a64bfd3c13808d94b55b6b6acef2ee", "8d7495d9008066505ed00ce8198af82bfa5a6b4c08768b4c9fb3aa4eb0b0cca2", "15800849a53349508cb382959527f6c3cf1a46158ff1e6e2316b7dea7967e35f", "7a792f0f4a2b731781d1b244b2a57947f1a2e32900a1c0793449f9f7ae18a7b7", "5e517c2832c9deaa7df77c7bad4d20fd6eda2b7815e155e68bc48238fac1416f", "9f51a14f0019c72bd1d472706d8c80a18c1873c6a0663e754b60eae8094483d7", "7d2fabb565122521d22ba99fed9e5be6a458fbc93156d54db27d97a00b8c3a97", "786c9e412a7db4ec278891fa534caa9a1d1a028c631c6f3aeb9c4d96ad895c36", "3bd6341d40641c2632a5a0cd7a63553a04e251efd7195897a1d27e02a7a8bfde", "31efd1f5fb57b8cff0318d77a1a9e8d67e1d1c8d18ce90f99c3a240dff48cdc8", "d9de3e1156ce1380150948acbcfecd99c96e7f4b0bc97745f4681593d017f74f", "427a2201e09f9583cd990c03b81b58148c297d474a3b50f498d83b1c7a9414cd", "bfaca11596d3dec406a9fcf5d97536516dfe7f0e3b12078428a7e1700e25218a", "351c4770a097248a650008152d0cab5825d048bef770da7f3364f59d1e721bc0", "ee00f205d1486b2be7381d962bd2867263758e880529e4e2bfedfa613bbc0e71", "6aa3b6418d89e3348e4859c823ef4d6d7cd46aa7f7e77aba586c4214d760d8f8", - ) -} - -/* -The regression test for the following invalid kademlia edge case. - -Addresses used in this test are discovered as part of the simulation network -in higher level tests for streaming. They were generated randomly. - -========================================================================= -Mon Apr 9 19:16:25 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 9a90fe -population: 8 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 -000 2 72ef 4e6c | 24 0b1e (0) 0d66 (0) 17f5 (0) 17e8 (0) -001 2 fc2b fa47 | 13 fa47 (0) fc2b (0) fffd (0) ecef (0) -002 2 b847 afa8 | 6 afa8 (0) ad77 (0) bb7c (0) b847 (0) -003 0 | 0 -004 0 | 4 91fc (0) 957d (0) 9482 (0) 949a (0) -============ DEPTH: 5 ========================================== -005 1 9ccf | 1 9ccf (0) -006 0 | 0 -007 1 9bb2 | 1 9bb2 (0) -008 0 | 0 -009 0 | 0 -010 0 | 0 -011 0 | 0 -012 0 | 0 -013 0 | 0 -014 0 | 0 -015 0 | 0 -========================================================================= -*/ -func TestKademliaCase4(t *testing.T) { - testKademliaCase(t, - "9a90fe3506277244549064b8c3276abb06284a199d9063a97331947f2b7da7f4", - "c19359eddef24b7be1a833b4475f212cd944263627a53f9ef4837d106c247730", "fc2b6fef99ef947f7e57c3df376891769e2a2fd83d2b8e634e0fc1e91eaa080c", "ecefc0e1a8ea7bb4b48c469e077401fce175dd75294255b96c4e54f6a2950a55", "bb7ce598efc056bba343cc2614aa3f67a575557561290b44c73a63f8f433f9f7", "55fbee6ca52dfd7f0be0db969ee8e524b654ab4f0cce7c05d83887d7d2a15460", "afa852b6b319998c6a283cc0c82d2f5b8e9410075d7700f3012761f1cfbd0f76", "36c370cfb63f2087971ba6e58d7585b04e16b8f0da335efb91554c2dd8fe191c", "6be41e029985edebc901fb77fc4fb65516b6d85086e2a98bfa3159c99391e585", "dd3cfc72ea553e7d2b28f0037a65646b30955b929d29ba4c40f4a2a811248e77", "da3a8f18e09c7b0ca235c4e33e1441a5188f1df023138bf207753ee63e768f7d", "de9e3ab4dc572d54a2d4b878329fd832bb51a149f4ce167316eeb177b61e7e01", "4e6c1ecde6ed917706257fe020a1d02d2e9d87fca4c85f0f7b132491008c5032", "72ef04b77a070e13463b3529dd312bcacfb7a12d20dc597f5ec3de0501e9b834", "3fef57186675d524ab8bb1f54ba8cb68610babca1247c0c46dbb60aed003c69d", "1d8e6b71f7a052865d6558d4ba44ad5fab7b908cc1badf5766822e1c20d0d823", "6be2f2b4ffa173014d4ec7df157d289744a2bda54bb876b264ccfa898a0da315", "b0ba3fff8643f9985c744327b0c4c869763509fd5da2de9a80a4a0a082021255", "9ccf40b9406ba2e6567101fb9b4e5334a9ec74263eff47267da266ba45e6c158", "d7347f02c180a448e60f73931845062ce00048750b584790278e9c93ef31ad81", "b68c6359a22b3bee6fecb8804311cfd816648ea31d530c9fb48e477e029d707a", "0d668a18ad7c2820214df6df95a6c855ce19fb1cb765f8ca620e45db76686d37", "3fbd2663bff65533246f1fabb9f38086854c6218aeb3dc9ac6ac73d4f0988f91", "949aa5719ca846052bfaa1b38c97b6eca3df3e24c0e0630042c6bccafbb4cdb5", "77b8a2b917bef5d54f3792183b014cca7798f713ff14fe0b2ac79b4c9f6f996d", "17e853cbd8dc00cba3cd9ffeb36f26a9f41a0eb92f80b62c2cda16771c935388", "5f682ed7a8cf2f98387c3def7c97f9f05ae39e39d393eeca3cf621268d6347f8", "ad77487eaf11fd8084ba4517a51766eb0e5b77dd3492dfa79aa3a2802fb29d20", "d247cfcacf9a8200ebaddf639f8c926ab0a001abe682f40df3785e80ed124e91", "195589442e11907eede1ee6524157f1125f68399f3170c835ff81c603b069f6c", "5b5ca0a67f3c54e7d3a6a862ef56168ec9ed1f4945e6c24de6d336b2be2e6f8c", "56430e4caa253015f1f998dce4a48a88af1953f68e94eca14f53074ae9c3e467", "0b1eed6a5bf612d1d8e08f5c546f3d12e838568fd3aa43ed4c537f10c65545d6", "7058db19a56dfff01988ac4a62e1310597f9c8d7ebde6890dadabf047d722d39", "b847380d6888ff7cd11402d086b19eccc40950b52c9d67e73cb4f8462f5df078", "df6c048419a2290ab546d527e9eeba349e7f7e1759bafe4adac507ce60ef9670", "91fc5b4b24fc3fbfea7f9a3d0f0437cb5733c0c2345d8bdffd7048d6e3b8a37b", "957d8ea51b37523952b6f5ae95462fcd4aed1483ef32cc80b69580aaeee03606", "efa82e4e91ad9ab781977400e9ac0bb9de7389aaedebdae979b73d1d3b8d72b0", "7400c9f3f3fc0cc6fe8cc37ab24b9771f44e9f78be913f73cd35fc4be030d6bd", "9bb28f4122d61f7bb56fe27ef706159fb802fef0f5de9dfa32c9c5b3183235f1", "40a8de6e98953498b806614532ea4abf8b99ad7f9719fb68203a6eae2efa5b2a", "412de0b218b8f7dcacc9205cd16ffb4eca5b838f46a2f4f9f534026061a47308", "17f56ecad51075080680ad9faa0fd8946b824d3296ddb20be07f9809fe8d1c5a", "fffd4e7ae885a41948a342b6647955a7ec8a8039039f510cff467ef597675457", "35e78e11b5ac46a29dd04ab0043136c3291f4ca56cb949ace33111ed56395463", "94824fc80230af82077c83bfc01dc9675b1f9d3d538b1e5f41c21ac753598691", "fa470ae314ca3fce493f21b423eef2a49522e09126f6f2326fa3c9cac0b344f7", "7078860b5b621b21ac7b95f9fc4739c8235ce5066a8b9bd7d938146a34fa88ec", "eea53560f0428bfd2eca4f86a5ce9dec5ff1309129a975d73465c1c9e9da71d1", - ) -} - -/* -The regression test for the following invalid kademlia edge case. - -Addresses used in this test are discovered as part of the simulation network -in higher level tests for streaming. They were generated randomly. - -========================================================================= -Mon Apr 9 19:25:18 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 5dd5c7 -population: 13 (49), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4 -000 2 e528 fad0 | 22 fad0 (0) e528 (0) e3bb (0) ed13 (0) -001 3 3f30 18e0 1dd3 | 7 3f30 (0) 23db (0) 10b6 (0) 18e0 (0) -002 4 7c54 7804 61e4 60f9 | 10 61e4 (0) 60f9 (0) 636c (0) 7186 (0) -003 2 40ae 4bae | 5 4bae (0) 4d5c (0) 403a (0) 40ae (0) -004 0 | 0 -005 0 | 3 5808 (0) 5a0e (0) 5bdb (0) -============ DEPTH: 6 ========================================== -006 2 5f14 5f61 | 2 5f14 (0) 5f61 (0) -007 0 | 0 -008 0 | 0 -009 0 | 0 -010 0 | 0 -011 0 | 0 -012 0 | 0 -013 0 | 0 -014 0 | 0 -015 0 | 0 -========================================================================= -*/ -func TestKademliaCase5(t *testing.T) { - testKademliaCase(t, - "5dd5c77dd9006a800478fcebb02d48d4036389e7d3c8f6a83b97dbad13f4c0a9", - "78fafa0809929a1279ece089a51d12457c2d8416dff859aeb2ccc24bb50df5ec", "1dd39b1257e745f147cbbc3cadd609ccd6207c41056dbc4254bba5d2527d3ee5", "5f61dd66d4d94aec8fcc3ce0e7885c7edf30c43143fa730e2841c5d28e3cd081", "8aa8b0472cb351d967e575ad05c4b9f393e76c4b01ef4b3a54aac5283b78abc9", "4502f385152a915b438a6726ce3ea9342e7a6db91a23c2f6bee83a885ed7eb82", "718677a504249db47525e959ef1784bed167e1c46f1e0275b9c7b588e28a3758", "7c54c6ed1f8376323896ed3a4e048866410de189e9599dd89bf312ca4adb96b5", "18e03bd3378126c09e799a497150da5c24c895aedc84b6f0dbae41fc4bac081a", "23db76ac9e6e58d9f5395ca78252513a7b4118b4155f8462d3d5eec62486cadc", "40ae0e8f065e96c7adb7fa39505136401f01780481e678d718b7f6dbb2c906ec", "c1539998b8bae19d339d6bbb691f4e9daeb0e86847545229e80fe0dffe716e92", "ed139d73a2699e205574c08722ca9f030ad2d866c662f1112a276b91421c3cb9", "5bdb19584b7a36d09ca689422ef7e6bb681b8f2558a6b2177a8f7c812f631022", "636c9de7fe234ffc15d67a504c69702c719f626c17461d3f2918e924cd9d69e2", "de4455413ff9335c440d52458c6544191bd58a16d85f700c1de53b62773064ea", "de1963310849527acabc7885b6e345a56406a8f23e35e436b6d9725e69a79a83", "a80a50a467f561210a114cba6c7fb1489ed43a14d61a9edd70e2eb15c31f074d", "7804f12b8d8e6e4b375b242058242068a3809385e05df0e64973cde805cf729c", "60f9aa320c02c6f2e6370aa740cf7cea38083fa95fca8c99552cda52935c1520", "d8da963602390f6c002c00ce62a84b514edfce9ebde035b277a957264bb54d21", "8463d93256e026fe436abad44697152b9a56ac8e06a0583d318e9571b83d073c", "9a3f78fcefb9a05e40a23de55f6153d7a8b9d973ede43a380bf46bb3b3847de1", "e3bb576f4b3760b9ca6bff59326f4ebfc4a669d263fb7d67ab9797adea54ed13", "4d5cdbd6dcca5bdf819a0fe8d175dc55cc96f088d37462acd5ea14bc6296bdbe", "5a0ed28de7b5258c727cb85447071c74c00a5fbba9e6bc0393bc51944d04ab2a", "61e4ddb479c283c638f4edec24353b6cc7a3a13b930824aad016b0996ca93c47", "7e3610868acf714836cafaaa7b8c009a9ac6e3a6d443e5586cf661530a204ee2", "d74b244d4345d2c86e30a097105e4fb133d53c578320285132a952cdaa64416e", "cfeed57d0f935bfab89e3f630a7c97e0b1605f0724d85a008bbfb92cb47863a8", "580837af95055670e20d494978f60c7f1458dc4b9e389fc7aa4982b2aca3bce3", "df55c0c49e6c8a83d82dfa1c307d3bf6a20e18721c80d8ec4f1f68dc0a137ced", "5f149c51ce581ba32a285439a806c063ced01ccd4211cd024e6a615b8f216f95", "1eb76b00aeb127b10dd1b7cd4c3edeb4d812b5a658f0feb13e85c4d2b7c6fe06", "7a56ba7c3fb7cbfb5561a46a75d95d7722096b45771ec16e6fa7bbfab0b35dfe", "4bae85ad88c28470f0015246d530adc0cd1778bdd5145c3c6b538ee50c4e04bd", "afd1892e2a7145c99ec0ebe9ded0d3fec21089b277a68d47f45961ec5e39e7e0", "953138885d7b36b0ef79e46030f8e61fd7037fbe5ce9e0a94d728e8c8d7eab86", "de761613ef305e4f628cb6bf97d7b7dc69a9d513dc233630792de97bcda777a6", "3f3087280063d09504c084bbf7fdf984347a72b50d097fd5b086ffabb5b3fb4c", "7d18a94bb1ebfdef4d3e454d2db8cb772f30ca57920dd1e402184a9e598581a0", "a7d6fbdc9126d9f10d10617f49fb9f5474ffe1b229f76b7dd27cebba30eccb5d", "fad0246303618353d1387ec10c09ee991eb6180697ed3470ed9a6b377695203d", "1cf66e09ea51ee5c23df26615a9e7420be2ac8063f28f60a3bc86020e94fe6f3", "8269cdaa153da7c358b0b940791af74d7c651cd4d3f5ed13acfe6d0f2c539e7f", "90d52eaaa60e74bf1c79106113f2599471a902d7b1c39ac1f55b20604f453c09", "9788fd0c09190a3f3d0541f68073a2f44c2fcc45bb97558a7c319f36c25a75b3", "10b68fc44157ecfdae238ee6c1ce0333f906ad04d1a4cb1505c8e35c3c87fbb0", "e5284117fdf3757920475c786e0004cb00ba0932163659a89b36651a01e57394", "403ad51d911e113dcd5f9ff58c94f6d278886a2a4da64c3ceca2083282c92de3", - ) -} - func newTestDiscoveryPeer(addr pot.Address, kad *Kademlia) *Peer { rw := &p2p.MsgPipeRW{} p := p2p.NewPeer(enode.ID{}, "foo", []p2p.Cap{}) diff --git a/swarm/network/networkid_test.go b/swarm/network/networkid_test.go index 99890118fc..8e68e5b521 100644 --- a/swarm/network/networkid_test.go +++ b/swarm/network/networkid_test.go @@ -76,13 +76,12 @@ func TestNetworkID(t *testing.T) { if err != nil { t.Fatalf("Error setting up network: %v", err) } - defer func() { - //shutdown the snapshot network - log.Trace("Shutting down network") - net.Shutdown() - }() //let's sleep to ensure all nodes are connected time.Sleep(1 * time.Second) + // shutdown the the network to avoid race conditions + // on accessing kademlias global map while network nodes + // are accepting messages + net.Shutdown() //for each group sharing the same network ID... for _, netIDGroup := range nodeMap { log.Trace("netIDGroup size", "size", len(netIDGroup)) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index a4b29239ce..6f8eadad2a 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -67,6 +67,7 @@ type BzzConfig struct { HiveParams *HiveParams NetworkID uint64 LightNode bool + BootnodeMode bool } // Bzz is the swarm protocol bundle @@ -87,7 +88,7 @@ type Bzz struct { // * overlay driver // * peer store func NewBzz(config *BzzConfig, kad *Kademlia, store state.Store, streamerSpec *protocols.Spec, streamerRun func(*BzzPeer) error) *Bzz { - return &Bzz{ + bzz := &Bzz{ Hive: NewHive(config.HiveParams, kad, store), NetworkID: config.NetworkID, LightNode: config.LightNode, @@ -96,6 +97,13 @@ func NewBzz(config *BzzConfig, kad *Kademlia, store state.Store, streamerSpec *p streamerRun: streamerRun, streamerSpec: streamerSpec, } + + if config.BootnodeMode { + bzz.streamerRun = nil + bzz.streamerSpec = nil + } + + return bzz } // UpdateLocalAddr updates underlayaddress of the running node @@ -168,7 +176,7 @@ func (b *Bzz) APIs() []rpc.API { func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(*p2p.Peer, p2p.MsgReadWriter) error { return func(p *p2p.Peer, rw p2p.MsgReadWriter) error { // wait for the bzz protocol to perform the handshake - handshake, _ := b.GetHandshake(p.ID()) + handshake, _ := b.GetOrCreateHandshake(p.ID()) defer b.removeHandshake(p.ID()) select { case <-handshake.done: @@ -213,7 +221,7 @@ func (b *Bzz) performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error // runBzz is the p2p protocol run function for the bzz base protocol // that negotiates the bzz handshake func (b *Bzz) runBzz(p *p2p.Peer, rw p2p.MsgReadWriter) error { - handshake, _ := b.GetHandshake(p.ID()) + handshake, _ := b.GetOrCreateHandshake(p.ID()) if !<-handshake.init { return fmt.Errorf("%08x: bzz already started on peer %08x", b.localAddr.Over()[:4], p.ID().Bytes()[:4]) } @@ -303,7 +311,7 @@ func (b *Bzz) removeHandshake(peerID enode.ID) { } // GetHandshake returns the bzz handhake that the remote peer with peerID sent -func (b *Bzz) GetHandshake(peerID enode.ID) (*HandshakeMsg, bool) { +func (b *Bzz) GetOrCreateHandshake(peerID enode.ID) (*HandshakeMsg, bool) { b.mtx.Lock() defer b.mtx.Unlock() handshake, found := b.handshakes[peerID] diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index 58477a7b8c..64ce7ba4ab 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" @@ -82,7 +83,7 @@ func newBzzBaseTester(t *testing.T, n int, addr *BzzAddr, spec *protocols.Spec, return srv(&BzzPeer{Peer: protocols.NewPeer(p, rw, spec), BzzAddr: NewAddr(p.Node())}) } - s := p2ptest.NewProtocolTester(t, addr.ID(), n, protocol) + s := p2ptest.NewProtocolTester(addr.ID(), n, protocol) for _, node := range s.Nodes { cs[node.ID().String()] = make(chan bool) @@ -115,9 +116,9 @@ func newBzz(addr *BzzAddr, lightNode bool) *Bzz { return bzz } -func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr, lightNode bool) *bzzTester { +func newBzzHandshakeTester(n int, addr *BzzAddr, lightNode bool) *bzzTester { bzz := newBzz(addr, lightNode) - pt := p2ptest.NewProtocolTester(t, addr.ID(), n, bzz.runBzz) + pt := p2ptest.NewProtocolTester(addr.ID(), n, bzz.runBzz) return &bzzTester{ addr: addr, @@ -165,7 +166,7 @@ func correctBzzHandshake(addr *BzzAddr, lightNode bool) *HandshakeMsg { func TestBzzHandshakeNetworkIDMismatch(t *testing.T) { lightNode := false addr := RandomAddr() - s := newBzzHandshakeTester(t, 1, addr, lightNode) + s := newBzzHandshakeTester(1, addr, lightNode) node := s.Nodes[0] err := s.testHandshake( @@ -182,7 +183,7 @@ func TestBzzHandshakeNetworkIDMismatch(t *testing.T) { func TestBzzHandshakeVersionMismatch(t *testing.T) { lightNode := false addr := RandomAddr() - s := newBzzHandshakeTester(t, 1, addr, lightNode) + s := newBzzHandshakeTester(1, addr, lightNode) node := s.Nodes[0] err := s.testHandshake( @@ -199,7 +200,7 @@ func TestBzzHandshakeVersionMismatch(t *testing.T) { func TestBzzHandshakeSuccess(t *testing.T) { lightNode := false addr := RandomAddr() - s := newBzzHandshakeTester(t, 1, addr, lightNode) + s := newBzzHandshakeTester(1, addr, lightNode) node := s.Nodes[0] err := s.testHandshake( @@ -224,7 +225,8 @@ func TestBzzHandshakeLightNode(t *testing.T) { for _, test := range lightNodeTests { t.Run(test.name, func(t *testing.T) { randomAddr := RandomAddr() - pt := newBzzHandshakeTester(t, 1, randomAddr, false) + pt := newBzzHandshakeTester(1, randomAddr, false) + node := pt.Nodes[0] addr := NewAddr(node) @@ -237,8 +239,14 @@ func TestBzzHandshakeLightNode(t *testing.T) { t.Fatal(err) } - if pt.bzz.handshakes[node.ID()].LightNode != test.lightNode { - t.Fatalf("peer LightNode flag is %v, should be %v", pt.bzz.handshakes[node.ID()].LightNode, test.lightNode) + select { + + case <-pt.bzz.handshakes[node.ID()].done: + if pt.bzz.handshakes[node.ID()].LightNode != test.lightNode { + t.Fatalf("peer LightNode flag is %v, should be %v", pt.bzz.handshakes[node.ID()].LightNode, test.lightNode) + } + case <-time.After(10 * time.Second): + t.Fatal("test timeout") } }) } diff --git a/swarm/network/simulation/kademlia_test.go b/swarm/network/simulation/kademlia_test.go index 36b244d3d0..bbc93ee8ca 100644 --- a/swarm/network/simulation/kademlia_test.go +++ b/swarm/network/simulation/kademlia_test.go @@ -28,8 +28,6 @@ import ( ) func TestWaitTillHealthy(t *testing.T) { - t.Skip("WaitTillHealthy depends on discovery, which relies on a reliable SuggestPeer, which is not reliable") - sim := New(map[string]ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { addr := network.NewAddr(ctx.Config.Node()) diff --git a/swarm/network/simulations/discovery/discovery_test.go b/swarm/network/simulations/discovery/discovery_test.go index e5121c4775..e695bc4ac0 100644 --- a/swarm/network/simulations/discovery/discovery_test.go +++ b/swarm/network/simulations/discovery/discovery_test.go @@ -18,16 +18,12 @@ package discovery import ( "context" - "encoding/json" - "errors" "flag" "fmt" "io/ioutil" - "math/rand" "os" "path" "strings" - "sync" "testing" "time" @@ -86,12 +82,10 @@ func getDbStore(nodeID string) (*state.DBStore, error) { } var ( - nodeCount = flag.Int("nodes", 10, "number of nodes to create (default 10)") - initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") - snapshotFile = flag.String("snapshot", "", "path to create snapshot file in") - loglevel = flag.Int("loglevel", 3, "verbosity of logs") - rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") - serviceOverride = flag.String("services", "", "remove or add services to the node snapshot; prefix with \"+\" to add, \"-\" to remove; example: +pss,-discovery") + nodeCount = flag.Int("nodes", 32, "number of nodes to create (default 32)") + initCount = flag.Int("conns", 1, "number of originally connected peers (default 1)") + loglevel = flag.Int("loglevel", 3, "verbosity of logs") + rawlog = flag.Bool("rawlog", false, "remove terminal formatting from logs") ) func init() { @@ -157,7 +151,6 @@ func testDiscoverySimulationSimAdapter(t *testing.T, nodes, conns int) { } func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) { - t.Skip("discovery tests depend on suggestpeer, which is unreliable after kademlia depth change.") startedAt := time.Now() result, err := discoverySimulation(nodes, conns, adapter) if err != nil { @@ -185,7 +178,6 @@ func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.No } func testDiscoveryPersistenceSimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) map[int][]byte { - t.Skip("discovery tests depend on suggestpeer, which is unreliable after kademlia depth change.") persistenceEnabled = true discoveryEnabled = true @@ -247,25 +239,14 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul action := func(ctx context.Context) error { return nil } - wg := sync.WaitGroup{} for i := range ids { // collect the overlay addresses, to addrs = append(addrs, ids[i].Bytes()) - for j := 0; j < conns; j++ { - var k int - if j == 0 { - k = (i + 1) % len(ids) - } else { - k = rand.Intn(len(ids)) - } - wg.Add(1) - go func(i, k int) { - defer wg.Done() - net.Connect(ids[i], ids[k]) - }(i, k) - } } - wg.Wait() + err := net.ConnectNodesChain(nil) + if err != nil { + return nil, err + } log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) // construct the peer pot, so that kademlia health can be checked ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) @@ -286,10 +267,10 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul } healthy := &network.Health{} - if err := client.Call(&healthy, "hive_healthy", ppmap); err != nil { + if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil { return false, fmt.Errorf("error getting node health: %s", err) } - log.Info(fmt.Sprintf("node %4s healthy: connected nearest neighbours: %v, know nearest neighbours: %v,\n\n%v", id, healthy.ConnectNN, healthy.KnowNN, healthy.Hive)) + log.Debug(fmt.Sprintf("node %4s healthy: connected nearest neighbours: %v, know nearest neighbours: %v,\n\n%v", id, healthy.ConnectNN, healthy.KnowNN, healthy.Hive)) return healthy.KnowNN && healthy.ConnectNN, nil } @@ -309,40 +290,6 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul if result.Error != nil { return result, nil } - - if *snapshotFile != "" { - var err error - var snap *simulations.Snapshot - if len(*serviceOverride) > 0 { - var addServices []string - var removeServices []string - for _, osvc := range strings.Split(*serviceOverride, ",") { - if strings.Index(osvc, "+") == 0 { - addServices = append(addServices, osvc[1:]) - } else if strings.Index(osvc, "-") == 0 { - removeServices = append(removeServices, osvc[1:]) - } else { - panic("stick to the rules, you know what they are") - } - } - snap, err = net.SnapshotWithServices(addServices, removeServices) - } else { - snap, err = net.Snapshot() - } - - if err != nil { - return nil, errors.New("no shapshot dude") - } - jsonsnapshot, err := json.Marshal(snap) - if err != nil { - return nil, fmt.Errorf("corrupt json snapshot: %v", err) - } - log.Info("writing snapshot", "file", *snapshotFile) - err = ioutil.WriteFile(*snapshotFile, jsonsnapshot, 0755) - if err != nil { - return nil, err - } - } return result, nil } @@ -405,7 +352,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt healthy := &network.Health{} addr := id.String() ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) - if err := client.Call(&healthy, "hive_healthy", ppmap); err != nil { + if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil { return fmt.Errorf("error getting node health: %s", err) } @@ -457,23 +404,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt return nil } - //connects in a chain - wg := sync.WaitGroup{} - //connects in a ring - for i := range ids { - for j := 1; j <= conns; j++ { - k := (i + j) % len(ids) - if k == i { - k = (k + 1) % len(ids) - } - wg.Add(1) - go func(i, k int) { - defer wg.Done() - net.Connect(ids[i], ids[k]) - }(i, k) - } - } - wg.Wait() + net.ConnectNodesChain(nil) log.Debug(fmt.Sprintf("nodes: %v", len(addrs))) // construct the peer pot, so that kademlia health can be checked check := func(ctx context.Context, id enode.ID) (bool, error) { @@ -494,7 +425,7 @@ func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapt healthy := &network.Health{} ppmap := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs) - if err := client.Call(&healthy, "hive_healthy", ppmap); err != nil { + if err := client.Call(&healthy, "hive_healthy", ppmap[common.Bytes2Hex(id.Bytes())]); err != nil { return false, fmt.Errorf("error getting node health: %s", err) } log.Info(fmt.Sprintf("node %4s healthy: got nearest neighbours: %v, know nearest neighbours: %v", id, healthy.ConnectNN, healthy.KnowNN)) diff --git a/swarm/network/simulations/overlay_test.go b/swarm/network/simulations/overlay_test.go index 6ccdb5ce25..e56b646503 100644 --- a/swarm/network/simulations/overlay_test.go +++ b/swarm/network/simulations/overlay_test.go @@ -43,7 +43,6 @@ var ( //It also provides a documentation on the steps needed by frontends //to use the simulations func TestOverlaySim(t *testing.T) { - t.Skip("Test is flaky, see: https://github.com/ethersphere/go-ethereum/issues/592") //start the simulation log.Info("Start simulation backend") //get the simulation networ; needed to subscribe for up events diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 29b917d39c..7b29626084 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -27,7 +27,6 @@ import ( "os" "strings" "sync/atomic" - "testing" "time" "github.com/ethereum/go-ethereum/log" @@ -67,7 +66,7 @@ func init() { log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) } -func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { +func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) { // setup addr := network.RandomAddr() // tested peers peer address to := network.NewKademlia(addr.OAddr, network.NewKadParams()) @@ -102,7 +101,7 @@ func newStreamerTester(t *testing.T, registryOptions *RegistryOptions) (*p2ptest streamer.Close() removeDataDir() } - protocolTester := p2ptest.NewProtocolTester(t, addr.ID(), 1, streamer.runProtocol) + protocolTester := p2ptest.NewProtocolTester(addr.ID(), 1, streamer.runProtocol) err = waitForPeers(streamer, 1*time.Second, 1) if err != nil { diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index e1a13fe8d5..c9a8dc57ac 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -255,8 +255,15 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) ( return true } sp = d.getPeer(id) + // sp is nil, when we encounter a peer that is not registered for delivery, i.e. doesn't support the `stream` protocol if sp == nil { - //log.Warn("Delivery.RequestFromPeers: peer not found", "id", id) + return true + } + // nodes that do not provide stream protocol + // should not be requested, e.g. bootnodes + if !p.HasCap("stream") { + // TODO: if we have no errors, delete this if + log.Error("Delivery.RequestFromPeers: peer doesn't have stream cap. we should have returned at sp == nil") return true } spID = &id diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index 70d3829b3f..6f6988fa4b 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -48,7 +48,7 @@ func TestStreamerRetrieveRequest(t *testing.T) { Retrieval: RetrievalClientOnly, Syncing: SyncingDisabled, } - tester, streamer, _, teardown, err := newStreamerTester(t, regOpts) + tester, streamer, _, teardown, err := newStreamerTester(regOpts) defer teardown() if err != nil { t.Fatal(err) @@ -97,7 +97,7 @@ func TestStreamerRetrieveRequest(t *testing.T) { //Test requesting a chunk from a peer then issuing a "empty" OfferedHashesMsg (no hashes available yet) //Should time out as the peer does not have the chunk (no syncing happened previously) func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + tester, streamer, _, teardown, err := newStreamerTester(&RegistryOptions{ Retrieval: RetrievalEnabled, Syncing: SyncingDisabled, //do no syncing }) @@ -169,7 +169,7 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { // upstream request server receives a retrieve Request and responds with // offered hashes or delivery if skipHash is set to true func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { - tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ + tester, streamer, localStore, teardown, err := newStreamerTester(&RegistryOptions{ Retrieval: RetrievalEnabled, Syncing: SyncingDisabled, }) @@ -285,7 +285,7 @@ func TestRequestFromPeers(t *testing.T) { addr := network.RandomAddr() to := network.NewKademlia(addr.OAddr, network.NewKadParams()) delivery := NewDelivery(to, nil) - protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", nil), nil, nil) + protocolsPeer := protocols.NewPeer(p2p.NewPeer(dummyPeerID, "dummy", []p2p.Cap{{Name: "stream"}}), nil, nil) peer := network.NewPeer(&network.BzzPeer{ BzzAddr: network.RandomAddr(), LightNode: false, @@ -359,7 +359,7 @@ func TestRequestFromPeersWithLightNode(t *testing.T) { } func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { - tester, streamer, localStore, teardown, err := newStreamerTester(t, &RegistryOptions{ + tester, streamer, localStore, teardown, err := newStreamerTester(&RegistryOptions{ Retrieval: RetrievalDisabled, Syncing: SyncingDisabled, }) diff --git a/swarm/network/stream/lightnode_test.go b/swarm/network/stream/lightnode_test.go index 65cde2411c..043f2ea779 100644 --- a/swarm/network/stream/lightnode_test.go +++ b/swarm/network/stream/lightnode_test.go @@ -28,7 +28,7 @@ func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) { Retrieval: RetrievalClientOnly, Syncing: SyncingDisabled, } - tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + tester, _, _, teardown, err := newStreamerTester(registryOptions) defer teardown() if err != nil { t.Fatal(err) @@ -67,7 +67,7 @@ func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) { Retrieval: RetrievalDisabled, Syncing: SyncingDisabled, } - tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + tester, _, _, teardown, err := newStreamerTester(registryOptions) defer teardown() if err != nil { t.Fatal(err) @@ -111,7 +111,7 @@ func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) { Retrieval: RetrievalDisabled, Syncing: SyncingRegisterOnly, } - tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + tester, _, _, teardown, err := newStreamerTester(registryOptions) defer teardown() if err != nil { t.Fatal(err) @@ -156,7 +156,7 @@ func TestLigthnodeRequestSubscriptionWithoutSync(t *testing.T) { Retrieval: RetrievalDisabled, Syncing: SyncingDisabled, } - tester, _, _, teardown, err := newStreamerTester(t, registryOptions) + tester, _, _, teardown, err := newStreamerTester(registryOptions) defer teardown() if err != nil { t.Fatal(err) diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index fb571c8566..0dafcc3b3a 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -516,6 +516,11 @@ func (r *Registry) requestPeerSubscriptions(kad *network.Kademlia, subs map[enod // nil as base takes the node's base; we need to pass 255 as `EachConn` runs // from deepest bins backwards kad.EachConn(nil, 255, func(p *network.Peer, po int) bool { + // nodes that do not provide stream protocol + // should not be subscribed, e.g. bootnodes + if !p.HasCap("stream") { + return true + } //if the peer's bin is shallower than the kademlia depth, //only the peer's bin should be subscribed if po < kadDepth { @@ -666,17 +671,16 @@ func peerStreamIntervalsKey(p *Peer, s Stream) string { return p.ID().String() + s.String() } -func (c client) AddInterval(start, end uint64) (err error) { +func (c *client) AddInterval(start, end uint64) (err error) { i := &intervals.Intervals{} - err = c.intervalsStore.Get(c.intervalsKey, i) - if err != nil { + if err = c.intervalsStore.Get(c.intervalsKey, i); err != nil { return err } i.Add(start, end) return c.intervalsStore.Put(c.intervalsKey, i) } -func (c client) NextInterval() (start, end uint64, err error) { +func (c *client) NextInterval() (start, end uint64, err error) { i := &intervals.Intervals{} err = c.intervalsStore.Get(c.intervalsKey, i) if err != nil { @@ -733,11 +737,7 @@ func (c *client) batchDone(p *Peer, req *OfferedHashesMsg, hashes []byte) error } return nil } - // TODO: make a test case for testing if the interval is added when the batch is done - if err := c.AddInterval(req.From, req.To); err != nil { - return err - } - return nil + return c.AddInterval(req.From, req.To) } func (c *client) close() { diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go index cdaeb92d0f..a41235e073 100644 --- a/swarm/network/stream/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -34,7 +34,7 @@ import ( ) func TestStreamerSubscribe(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(nil) defer teardown() if err != nil { t.Fatal(err) @@ -48,7 +48,7 @@ func TestStreamerSubscribe(t *testing.T) { } func TestStreamerRequestSubscription(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(nil) defer teardown() if err != nil { t.Fatal(err) @@ -139,7 +139,7 @@ func (self *testServer) Close() { } func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(nil) defer teardown() if err != nil { t.Fatal(err) @@ -232,7 +232,7 @@ func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { } func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(nil) defer teardown() if err != nil { t.Fatal(err) @@ -299,7 +299,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { } func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(nil) defer teardown() if err != nil { t.Fatal(err) @@ -365,7 +365,7 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) { } func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(nil) defer teardown() if err != nil { t.Fatal(err) @@ -409,7 +409,7 @@ func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) { } func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(nil) defer teardown() if err != nil { t.Fatal(err) @@ -472,7 +472,7 @@ func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) { } func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(nil) defer teardown() if err != nil { t.Fatal(err) @@ -537,7 +537,7 @@ func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) { } func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(nil) defer teardown() if err != nil { t.Fatal(err) @@ -636,7 +636,7 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { } func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) { - tester, streamer, _, teardown, err := newStreamerTester(t, nil) + tester, streamer, _, teardown, err := newStreamerTester(nil) defer teardown() if err != nil { t.Fatal(err) @@ -769,7 +769,7 @@ func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) { // leaving place for new streams. func TestMaxPeerServersWithUnsubscribe(t *testing.T) { var maxPeerServers = 6 - tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + tester, streamer, _, teardown, err := newStreamerTester(&RegistryOptions{ Retrieval: RetrievalDisabled, Syncing: SyncingDisabled, MaxPeerServers: maxPeerServers, @@ -845,7 +845,7 @@ func TestMaxPeerServersWithUnsubscribe(t *testing.T) { // error message exchange. func TestMaxPeerServersWithoutUnsubscribe(t *testing.T) { var maxPeerServers = 6 - tester, streamer, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + tester, streamer, _, teardown, err := newStreamerTester(&RegistryOptions{ MaxPeerServers: maxPeerServers, }) defer teardown() @@ -930,7 +930,7 @@ func TestMaxPeerServersWithoutUnsubscribe(t *testing.T) { //TestHasPriceImplementation is to check that the Registry has a //`Price` interface implementation func TestHasPriceImplementation(t *testing.T) { - _, r, _, teardown, err := newStreamerTester(t, &RegistryOptions{ + _, r, _, teardown, err := newStreamerTester(&RegistryOptions{ Retrieval: RetrievalDisabled, Syncing: SyncingDisabled, }) diff --git a/swarm/network/stream/visualized_snapshot_sync_sim_test.go b/swarm/network/stream/visualized_snapshot_sync_sim_test.go index 18b4c8fb0b..3b7d0d7435 100644 --- a/swarm/network/stream/visualized_snapshot_sync_sim_test.go +++ b/swarm/network/stream/visualized_snapshot_sync_sim_test.go @@ -154,8 +154,6 @@ func sendSimTerminatedEvent(sim *simulation.Simulation) { //It also sends some custom events so that the frontend //can visualize messages like SendOfferedMsg, WantedHashesMsg, DeliveryMsg func TestSnapshotSyncWithServer(t *testing.T) { - //t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") - //define a wrapper object to be able to pass around data wrapper := &netWrapper{} diff --git a/swarm/pss/notify/notify_test.go b/swarm/pss/notify/notify_test.go index 5c29f68e0f..95121b1d8f 100644 --- a/swarm/pss/notify/notify_test.go +++ b/swarm/pss/notify/notify_test.go @@ -51,6 +51,7 @@ func TestStart(t *testing.T) { ID: "0", DefaultService: "bzz", }) + defer net.Shutdown() leftNodeConf := adapters.RandomNodeConfig() leftNodeConf.Services = []string{"bzz", "pss"} leftNode, err := net.NewNodeWithConfig(leftNodeConf) @@ -128,7 +129,7 @@ func TestStart(t *testing.T) { defer rightSub.Unsubscribe() updateC := make(chan []byte) - updateMsg := []byte{} + var updateMsg []byte ctrlClient := NewController(psses[rightPub]) ctrlNotifier := NewController(psses[leftPub]) ctrlNotifier.NewNotifier("foo.eth", 2, updateC) @@ -145,17 +146,24 @@ func TestStart(t *testing.T) { if err != nil { t.Fatal(err) } + + copyOfUpdateMsg := make([]byte, len(updateMsg)) + copy(copyOfUpdateMsg, updateMsg) + ctrlClientError := make(chan error, 1) ctrlClient.Subscribe(rsrcName, pubkey, addrbytes, func(s string, b []byte) error { - if s != "foo.eth" || !bytes.Equal(updateMsg, b) { - t.Fatalf("unexpected result in client handler: '%s':'%x'", s, b) + if s != "foo.eth" || !bytes.Equal(copyOfUpdateMsg, b) { + ctrlClientError <- fmt.Errorf("unexpected result in client handler: '%s':'%x'", s, b) + } else { + log.Info("client handler receive", "s", s, "b", b) } - log.Info("client handler receive", "s", s, "b", b) return nil }) var inMsg *pss.APIMsg select { case inMsg = <-rmsgC: + case err := <-ctrlClientError: + t.Fatal(err) case <-ctx.Done(): t.Fatal(ctx.Err()) } diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index bee64b0df5..ee942303c5 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -138,10 +138,11 @@ type Pss struct { symKeyDecryptCacheCapacity int // max amount of symkeys to keep. // message handling - handlers map[Topic]map[*handler]bool // topic and version based pss payload handlers. See pss.Handle() - handlersMu sync.RWMutex - hashPool sync.Pool - topicHandlerCaps map[Topic]*handlerCaps // caches capabilities of each topic's handlers (see handlerCap* consts in types.go) + handlers map[Topic]map[*handler]bool // topic and version based pss payload handlers. See pss.Handle() + handlersMu sync.RWMutex + hashPool sync.Pool + topicHandlerCaps map[Topic]*handlerCaps // caches capabilities of each topic's handlers + topicHandlerCapsMu sync.RWMutex // process quitC chan struct{} @@ -307,6 +308,19 @@ func (p *Pss) PublicKey() *ecdsa.PublicKey { // SECTION: Message handling ///////////////////////////////////////////////////////////////////// +func (p *Pss) getTopicHandlerCaps(topic Topic) (hc *handlerCaps, found bool) { + p.topicHandlerCapsMu.RLock() + defer p.topicHandlerCapsMu.RUnlock() + hc, found = p.topicHandlerCaps[topic] + return +} + +func (p *Pss) setTopicHandlerCaps(topic Topic, hc *handlerCaps) { + p.topicHandlerCapsMu.Lock() + defer p.topicHandlerCapsMu.Unlock() + p.topicHandlerCaps[topic] = hc +} + // Links a handler function to a Topic // // All incoming messages with an envelope Topic matching the @@ -323,23 +337,28 @@ func (p *Pss) Register(topic *Topic, hndlr *handler) func() { if handlers == nil { handlers = make(map[*handler]bool) p.handlers[*topic] = handlers - log.Debug("registered handler", "caps", hndlr.caps) + log.Debug("registered handler", "capabilities", hndlr.caps) } if hndlr.caps == nil { hndlr.caps = &handlerCaps{} } handlers[hndlr] = true - if _, ok := p.topicHandlerCaps[*topic]; !ok { - p.topicHandlerCaps[*topic] = &handlerCaps{} + + capabilities, ok := p.getTopicHandlerCaps(*topic) + if !ok { + capabilities = &handlerCaps{} + p.setTopicHandlerCaps(*topic, capabilities) } + if hndlr.caps.raw { - p.topicHandlerCaps[*topic].raw = true + capabilities.raw = true } if hndlr.caps.prox { - p.topicHandlerCaps[*topic].prox = true + capabilities.prox = true } return func() { p.deregister(topic, hndlr) } } + func (p *Pss) deregister(topic *Topic, hndlr *handler) { p.handlersMu.Lock() defer p.handlersMu.Unlock() @@ -356,19 +375,12 @@ func (p *Pss) deregister(topic *Topic, hndlr *handler) { caps.prox = true } } - p.topicHandlerCaps[*topic] = caps + p.setTopicHandlerCaps(*topic, caps) return } delete(handlers, hndlr) } -// get all registered handlers for respective topics -func (p *Pss) getHandlers(topic Topic) map[*handler]bool { - p.handlersMu.RLock() - defer p.handlersMu.RUnlock() - return p.handlers[topic] -} - // Filters incoming messages for processing or forwarding. // Check if address partially matches // If yes, it CAN be for us, and we process it @@ -396,8 +408,8 @@ func (p *Pss) handlePssMsg(ctx context.Context, msg interface{}) error { // raw is simplest handler contingency to check, so check that first var isRaw bool if pssmsg.isRaw() { - if _, ok := p.topicHandlerCaps[psstopic]; ok { - if !p.topicHandlerCaps[psstopic].raw { + if capabilities, ok := p.getTopicHandlerCaps(psstopic); ok { + if !capabilities.raw { log.Debug("No handler for raw message", "topic", psstopic) return nil } @@ -410,8 +422,8 @@ func (p *Pss) handlePssMsg(ctx context.Context, msg interface{}) error { // - prox handler on message and we are in prox regardless of partial address match // store this result so we don't calculate again on every handler var isProx bool - if _, ok := p.topicHandlerCaps[psstopic]; ok { - isProx = p.topicHandlerCaps[psstopic].prox + if capabilities, ok := p.getTopicHandlerCaps(psstopic); ok { + isProx = capabilities.prox } isRecipient := p.isSelfPossibleRecipient(pssmsg, isProx) if !isRecipient { @@ -427,7 +439,6 @@ func (p *Pss) handlePssMsg(ctx context.Context, msg interface{}) error { } } return nil - } // Entry point to processing a message for which the current node can be the intended recipient. @@ -472,13 +483,22 @@ func (p *Pss) process(pssmsg *PssMsg, raw bool, prox bool) error { p.executeHandlers(psstopic, payload, from, raw, prox, asymmetric, keyid) return nil +} +// copy all registered handlers for respective topic in order to avoid data race or deadlock +func (p *Pss) getHandlers(topic Topic) (ret []*handler) { + p.handlersMu.RLock() + defer p.handlersMu.RUnlock() + for k := range p.handlers[topic] { + ret = append(ret, k) + } + return ret } func (p *Pss) executeHandlers(topic Topic, payload []byte, from PssAddress, raw bool, prox bool, asymmetric bool, keyid string) { handlers := p.getHandlers(topic) peer := p2p.NewPeer(enode.ID{}, fmt.Sprintf("%x", from), []p2p.Cap{}) - for h := range handlers { + for _, h := range handlers { if !h.caps.raw && raw { log.Warn("norawhandler") continue @@ -781,8 +801,8 @@ func (p *Pss) SendRaw(address PssAddress, topic Topic, msg []byte) error { // if we have a proxhandler on this topic // also deliver message to ourselves - if _, ok := p.topicHandlerCaps[topic]; ok { - if p.isSelfPossibleRecipient(pssMsg, true) && p.topicHandlerCaps[topic].prox { + if capabilities, ok := p.getTopicHandlerCaps(topic); ok { + if p.isSelfPossibleRecipient(pssMsg, true) && capabilities.prox { return p.process(pssMsg, true, true) } } @@ -883,8 +903,8 @@ func (p *Pss) send(to []byte, topic Topic, msg []byte, asymmetric bool, key []by if err != nil { return err } - if _, ok := p.topicHandlerCaps[topic]; ok { - if p.isSelfPossibleRecipient(pssMsg, true) && p.topicHandlerCaps[topic].prox { + if capabilities, ok := p.getTopicHandlerCaps(topic); ok { + if p.isSelfPossibleRecipient(pssMsg, true) && capabilities.prox { return p.process(pssMsg, true, true) } } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 1fe466f930..c87351323d 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -26,7 +26,6 @@ import ( "strconv" "strings" "testing" - "time" "github.com/ethereum/go-ethereum/common" ch "github.com/ethereum/go-ethereum/swarm/chunk" @@ -388,11 +387,11 @@ func testLDBStoreCollectGarbage(t *testing.T) { t.Fatal(err.Error()) } allChunks = append(allChunks, chunks...) + ldb.lock.RLock() log.Debug("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt, "cap", capacity, "n", n) + ldb.lock.RUnlock() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - waitGc(ctx, ldb) + waitGc(ldb) } // attempt gets on all put chunks @@ -466,6 +465,7 @@ func TestLDBStoreAddRemove(t *testing.T) { } func testLDBStoreRemoveThenCollectGarbage(t *testing.T) { + t.Skip("flaky with -race flag") params := strings.Split(t.Name(), "/") capacity, err := strconv.Atoi(params[2]) @@ -496,9 +496,7 @@ func testLDBStoreRemoveThenCollectGarbage(t *testing.T) { } } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - waitGc(ctx, ldb) + waitGc(ldb) // delete all chunks // (only count the ones actually deleted, the rest will have been gc'd) @@ -537,14 +535,14 @@ func testLDBStoreRemoveThenCollectGarbage(t *testing.T) { remaining -= putCount for putCount > 0 { ldb.Put(context.TODO(), chunks[puts]) + ldb.lock.RLock() log.Debug("ldbstore", "entrycnt", ldb.entryCnt, "accesscnt", ldb.accessCnt, "cap", capacity, "n", n, "puts", puts, "remaining", remaining, "roundtarget", roundTarget) + ldb.lock.RUnlock() puts++ putCount-- } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - waitGc(ctx, ldb) + waitGc(ldb) } // expect first surplus chunks to be missing, because they have the smallest access value @@ -597,9 +595,7 @@ func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { } // wait for garbage collection to kick in on the responsible actor - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - waitGc(ctx, ldb) + waitGc(ldb) var missing int for i, ch := range chunks[2 : capacity/2] { @@ -788,7 +784,10 @@ func TestCleanIndex(t *testing.T) { } } -func waitGc(ctx context.Context, ldb *LDBStore) { +// Note: waitGc does not guarantee that we wait 1 GC round; it only +// guarantees that if the GC is running we wait for that run to finish +// ticket: https://github.com/ethersphere/go-ethereum/issues/1151 +func waitGc(ldb *LDBStore) { <-ldb.gc.runC ldb.gc.runC <- struct{}{} } diff --git a/swarm/storage/netstore_test.go b/swarm/storage/netstore_test.go index 2ed3e0752d..a6a9f551ab 100644 --- a/swarm/storage/netstore_test.go +++ b/swarm/storage/netstore_test.go @@ -43,6 +43,7 @@ type mockNetFetcher struct { quit <-chan struct{} ctx context.Context hopCounts []uint8 + mu sync.Mutex } func (m *mockNetFetcher) Offer(ctx context.Context, source *enode.ID) { @@ -51,6 +52,9 @@ func (m *mockNetFetcher) Offer(ctx context.Context, source *enode.ID) { } func (m *mockNetFetcher) Request(ctx context.Context, hopCount uint8) { + m.mu.Lock() + defer m.mu.Unlock() + m.requestCalled = true var peers []Address m.peers.Range(func(key interface{}, _ interface{}) bool { diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index e5bd7a76a1..ed0f843b93 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -201,8 +201,6 @@ func (pc *PyramidChunker) decrementWorkerCount() { } func (pc *PyramidChunker) Split(ctx context.Context) (k Address, wait func(context.Context) error, err error) { - log.Debug("pyramid.chunker: Split()") - pc.wg.Add(1) pc.prepareChunks(ctx, false) @@ -235,7 +233,6 @@ func (pc *PyramidChunker) Split(ctx context.Context) (k Address, wait func(conte } func (pc *PyramidChunker) Append(ctx context.Context) (k Address, wait func(context.Context) error, err error) { - log.Debug("pyramid.chunker: Append()") // Load the right most unfinished tree chunks in every level pc.loadTree(ctx) @@ -283,8 +280,6 @@ func (pc *PyramidChunker) processor(ctx context.Context, id int64) { } func (pc *PyramidChunker) processChunk(ctx context.Context, id int64, job *chunkJob) { - log.Debug("pyramid.chunker: processChunk()", "id", id) - ref, err := pc.putter.Put(ctx, job.chunk) if err != nil { select { @@ -301,7 +296,6 @@ func (pc *PyramidChunker) processChunk(ctx context.Context, id int64, job *chunk } func (pc *PyramidChunker) loadTree(ctx context.Context) error { - log.Debug("pyramid.chunker: loadTree()") // Get the root chunk to get the total size chunkData, err := pc.getter.Get(ctx, Reference(pc.key)) if err != nil { @@ -386,7 +380,6 @@ func (pc *PyramidChunker) loadTree(ctx context.Context) error { } func (pc *PyramidChunker) prepareChunks(ctx context.Context, isAppend bool) { - log.Debug("pyramid.chunker: prepareChunks", "isAppend", isAppend) defer pc.wg.Done() chunkWG := &sync.WaitGroup{} diff --git a/swarm/swarm.go b/swarm/swarm.go index db52675fdb..9ece8bb669 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -56,7 +56,6 @@ import ( ) var ( - startTime time.Time updateGaugesPeriod = 5 * time.Second startCounter = metrics.NewRegisteredCounter("stack,start", nil) stopCounter = metrics.NewRegisteredCounter("stack,stop", nil) @@ -80,16 +79,16 @@ type Swarm struct { swap *swap.Swap stateStore *state.DBStore accountingMetrics *protocols.AccountingMetrics + startTime time.Time tracerClose io.Closer } -// creates a new swarm service instance +// NewSwarm creates a new swarm service instance // implements node.Service // If mockStore is not nil, it will be used as the storage for chunk data. // MockStore should be used only for testing. func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err error) { - if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroAddr) { return nil, fmt.Errorf("empty public key") } @@ -116,10 +115,11 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e config.HiveParams.Discovery = true bzzconfig := &network.BzzConfig{ - NetworkID: config.NetworkID, - OverlayAddr: common.FromHex(config.BzzKey), - HiveParams: config.HiveParams, - LightNode: config.LightNodeEnabled, + NetworkID: config.NetworkID, + OverlayAddr: common.FromHex(config.BzzKey), + HiveParams: config.HiveParams, + LightNode: config.LightNodeEnabled, + BootnodeMode: config.BootnodeMode, } self.stateStore, err = state.NewDBStore(filepath.Join(config.Path, "state-store.db")) @@ -344,7 +344,7 @@ Start is called when the stack is started */ // implements the node.Service interface func (self *Swarm) Start(srv *p2p.Server) error { - startTime = time.Now() + self.startTime = time.Now() self.tracerClose = tracing.Closer @@ -414,7 +414,7 @@ func (self *Swarm) periodicallyUpdateGauges() { } func (self *Swarm) updateGauges() { - uptimeGauge.Update(time.Since(startTime).Nanoseconds()) + uptimeGauge.Update(time.Since(self.startTime).Nanoseconds()) requestsCacheGauge.Update(int64(self.netStore.RequestsCacheLen())) } @@ -455,12 +455,16 @@ func (self *Swarm) Stop() error { return err } -// implements the node.Service interface -func (self *Swarm) Protocols() (protos []p2p.Protocol) { - protos = append(protos, self.bzz.Protocols()...) +// Protocols implements the node.Service interface +func (s *Swarm) Protocols() (protos []p2p.Protocol) { + if s.config.BootnodeMode { + protos = append(protos, s.bzz.Protocols()...) + } else { + protos = append(protos, s.bzz.Protocols()...) - if self.ps != nil { - protos = append(protos, self.ps.Protocols()...) + if s.ps != nil { + protos = append(protos, s.ps.Protocols()...) + } } return } diff --git a/swarm/version/version.go b/swarm/version/version.go index fb4c531110..9c72481104 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -41,6 +41,9 @@ var VersionWithMeta = func() string { return v }() +// Git SHA1 commit hash of the release, will be set by main.init() function +var GitCommit string + // ArchiveVersion holds the textual version string used for Swarm archives. // e.g. "0.3.0-dea1ce05" for stable releases, or // "0.3.1-unstable-21c059b6" for unstable releases diff --git a/tests/init.go b/tests/init.go index db0457b6d7..188cdffe9d 100644 --- a/tests/init.go +++ b/tests/init.go @@ -62,6 +62,18 @@ var Forks = map[string]*params.ChainConfig{ DAOForkBlock: big.NewInt(0), ByzantiumBlock: big.NewInt(0), ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(10000000), + }, + "ConstantinopleFix": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + DAOForkBlock: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), }, "FrontierToHomesteadAt5": { ChainID: big.NewInt(1), diff --git a/tests/rlp_test_util.go b/tests/rlp_test_util.go index 58ef8a6428..9069ec55a1 100644 --- a/tests/rlp_test_util.go +++ b/tests/rlp_test_util.go @@ -42,9 +42,22 @@ type RLPTest struct { Out string } +// FromHex returns the bytes represented by the hexadecimal string s. +// s may be prefixed with "0x". +// This is copy-pasted from bytes.go, which does not return the error +func FromHex(s string) ([]byte, error) { + if len(s) > 1 && (s[0:2] == "0x" || s[0:2] == "0X") { + s = s[2:] + } + if len(s)%2 == 1 { + s = "0" + s + } + return hex.DecodeString(s) +} + // Run executes the test. func (t *RLPTest) Run() error { - outb, err := hex.DecodeString(t.Out) + outb, err := FromHex(t.Out) if err != nil { return fmt.Errorf("invalid hex in Out") } diff --git a/tests/state_test.go b/tests/state_test.go index 964405382c..8b69da91f2 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -17,6 +17,7 @@ package tests import ( + "bufio" "bytes" "flag" "fmt" @@ -45,9 +46,12 @@ func TestState(t *testing.T) { st.skipLoad(`^stTransactionTest/OverflowGasRequire\.json`) // gasLimit > 256 bits st.skipLoad(`^stTransactionTest/zeroSigTransa[^/]*\.json`) // EIP-86 is not supported yet // Expected failures: - st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/EIP158`, "bug in test") - st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/Byzantium`, "bug in test") - st.fails(`^stRevertTest/RevertPrecompiledTouch.json/Constantinople`, "bug in test") + st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test") + st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/3`, "bug in test") + st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/0`, "bug in test") + st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/3`, "bug in test") + st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/0`, "bug in test") + st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/3`, "bug in test") st.walk(t, stateTestDir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { @@ -86,18 +90,19 @@ func withTrace(t *testing.T, gasLimit uint64, test func(vm.Config) error) { t.Log("gas limit too high for EVM trace") return } - tracer := vm.NewStructLogger(nil) + buf := new(bytes.Buffer) + w := bufio.NewWriter(buf) + tracer := vm.NewJSONLogger(&vm.LogConfig{DisableMemory: true}, w) err2 := test(vm.Config{Debug: true, Tracer: tracer}) if !reflect.DeepEqual(err, err2) { t.Errorf("different error for second run: %v", err2) } - buf := new(bytes.Buffer) - vm.WriteTrace(buf, tracer.StructLogs()) + w.Flush() if buf.Len() == 0 { t.Log("no EVM operation logs generated") } else { t.Log("EVM operation log:\n" + buf.String()) } - t.Logf("EVM output: 0x%x", tracer.Output()) - t.Logf("EVM error: %v", tracer.Error()) + //t.Logf("EVM output: 0x%x", tracer.Output()) + //t.Logf("EVM error: %v", tracer.Error()) } diff --git a/tests/testdata b/tests/testdata index c02a2a17c0..6b85703b56 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit c02a2a17c0288a255572b37dc7ec1fcb838b9dbf +Subproject commit 6b85703b568f4456582a00665d8a3e5c3b20b484 diff --git a/vendor/github.com/graph-gophers/graphql-go/Gopkg.lock b/vendor/github.com/graph-gophers/graphql-go/Gopkg.lock new file mode 100644 index 0000000000..4574275c5d --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/Gopkg.lock @@ -0,0 +1,25 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/opentracing/opentracing-go" + packages = [ + ".", + "ext", + "log" + ] + revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38" + version = "v1.0.2" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = ["context"] + revision = "f5dfe339be1d06f81b22525fe34671ee7d2c8904" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "f417062128566756a9360b1c13ada79bdeeb6bab1f53ee9147a3328d95c1653f" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/vendor/github.com/graph-gophers/graphql-go/Gopkg.toml b/vendor/github.com/graph-gophers/graphql-go/Gopkg.toml new file mode 100644 index 0000000000..62b9367998 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/Gopkg.toml @@ -0,0 +1,10 @@ +# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html +# for detailed Gopkg.toml documentation. + +[[constraint]] + name = "github.com/opentracing/opentracing-go" + version = "1.0.2" + +[prune] + go-tests = true + unused-packages = true diff --git a/vendor/github.com/graph-gophers/graphql-go/LICENSE b/vendor/github.com/graph-gophers/graphql-go/LICENSE new file mode 100644 index 0000000000..3907cecacf --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2016 Richard Musiol. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/graph-gophers/graphql-go/README.md b/vendor/github.com/graph-gophers/graphql-go/README.md new file mode 100644 index 0000000000..ef4b4639b5 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/README.md @@ -0,0 +1,100 @@ +# graphql-go [![Sourcegraph](https://sourcegraph.com/github.com/graph-gophers/graphql-go/-/badge.svg)](https://sourcegraph.com/github.com/graph-gophers/graphql-go?badge) [![Build Status](https://semaphoreci.com/api/v1/graph-gophers/graphql-go/branches/master/badge.svg)](https://semaphoreci.com/graph-gophers/graphql-go) [![GoDoc](https://godoc.org/github.com/graph-gophers/graphql-go?status.svg)](https://godoc.org/github.com/graph-gophers/graphql-go) + +

+ +The goal of this project is to provide full support of the [GraphQL draft specification](https://facebook.github.io/graphql/draft) with a set of idiomatic, easy to use Go packages. + +While still under heavy development (`internal` APIs are almost certainly subject to change), this library is +safe for production use. + +## Features + +- minimal API +- support for `context.Context` +- support for the `OpenTracing` standard +- schema type-checking against resolvers +- resolvers are matched to the schema based on method sets (can resolve a GraphQL schema with a Go interface or Go struct). +- handles panics in resolvers +- parallel execution of resolvers + +## Roadmap + +We're trying out the GitHub Project feature to manage `graphql-go`'s [development roadmap](https://github.com/graph-gophers/graphql-go/projects/1). +Feedback is welcome and appreciated. + +## (Some) Documentation + +### Basic Sample + +```go +package main + +import ( + "log" + "net/http" + + graphql "github.com/graph-gophers/graphql-go" + "github.com/graph-gophers/graphql-go/relay" +) + +type query struct{} + +func (_ *query) Hello() string { return "Hello, world!" } + +func main() { + s := ` + schema { + query: Query + } + type Query { + hello: String! + } + ` + schema := graphql.MustParseSchema(s, &query{}) + http.Handle("/query", &relay.Handler{Schema: schema}) + log.Fatal(http.ListenAndServe(":8080", nil)) +} +``` + +To test: +```sh +$ curl -XPOST -d '{"query": "{ hello }"}' localhost:8080/query +``` + +### Resolvers + +A resolver must have one method for each field of the GraphQL type it resolves. The method name has to be [exported](https://golang.org/ref/spec#Exported_identifiers) and match the field's name in a non-case-sensitive way. + +The method has up to two arguments: + +- Optional `context.Context` argument. +- Mandatory `*struct { ... }` argument if the corresponding GraphQL field has arguments. The names of the struct fields have to be [exported](https://golang.org/ref/spec#Exported_identifiers) and have to match the names of the GraphQL arguments in a non-case-sensitive way. + +The method has up to two results: + +- The GraphQL field's value as determined by the resolver. +- Optional `error` result. + +Example for a simple resolver method: + +```go +func (r *helloWorldResolver) Hello() string { + return "Hello world!" +} +``` + +The following signature is also allowed: + +```go +func (r *helloWorldResolver) Hello(ctx context.Context) (string, error) { + return "Hello world!", nil +} +``` + +### Community Examples + +[tonyghita/graphql-go-example](https://github.com/tonyghita/graphql-go-example) - A more "productionized" version of the Star Wars API example given in this repository. + +[deltaskelta/graphql-go-pets-example](https://github.com/deltaskelta/graphql-go-pets-example) - graphql-go resolving against a sqlite database + +[OscarYuen/go-graphql-starter](https://github.com/OscarYuen/go-graphql-starter) - a starter application integrated with dataloader, psql and basic authentication diff --git a/vendor/github.com/graph-gophers/graphql-go/errors/errors.go b/vendor/github.com/graph-gophers/graphql-go/errors/errors.go new file mode 100644 index 0000000000..fdfa62024d --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/errors/errors.go @@ -0,0 +1,41 @@ +package errors + +import ( + "fmt" +) + +type QueryError struct { + Message string `json:"message"` + Locations []Location `json:"locations,omitempty"` + Path []interface{} `json:"path,omitempty"` + Rule string `json:"-"` + ResolverError error `json:"-"` +} + +type Location struct { + Line int `json:"line"` + Column int `json:"column"` +} + +func (a Location) Before(b Location) bool { + return a.Line < b.Line || (a.Line == b.Line && a.Column < b.Column) +} + +func Errorf(format string, a ...interface{}) *QueryError { + return &QueryError{ + Message: fmt.Sprintf(format, a...), + } +} + +func (err *QueryError) Error() string { + if err == nil { + return "" + } + str := fmt.Sprintf("graphql: %s", err.Message) + for _, loc := range err.Locations { + str += fmt.Sprintf(" (line %d, column %d)", loc.Line, loc.Column) + } + return str +} + +var _ error = &QueryError{} diff --git a/vendor/github.com/graph-gophers/graphql-go/graphql.go b/vendor/github.com/graph-gophers/graphql-go/graphql.go new file mode 100644 index 0000000000..06ffd45997 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/graphql.go @@ -0,0 +1,205 @@ +package graphql + +import ( + "context" + "fmt" + + "encoding/json" + + "github.com/graph-gophers/graphql-go/errors" + "github.com/graph-gophers/graphql-go/internal/common" + "github.com/graph-gophers/graphql-go/internal/exec" + "github.com/graph-gophers/graphql-go/internal/exec/resolvable" + "github.com/graph-gophers/graphql-go/internal/exec/selected" + "github.com/graph-gophers/graphql-go/internal/query" + "github.com/graph-gophers/graphql-go/internal/schema" + "github.com/graph-gophers/graphql-go/internal/validation" + "github.com/graph-gophers/graphql-go/introspection" + "github.com/graph-gophers/graphql-go/log" + "github.com/graph-gophers/graphql-go/trace" +) + +// ParseSchema parses a GraphQL schema and attaches the given root resolver. It returns an error if +// the Go type signature of the resolvers does not match the schema. If nil is passed as the +// resolver, then the schema can not be executed, but it may be inspected (e.g. with ToJSON). +func ParseSchema(schemaString string, resolver interface{}, opts ...SchemaOpt) (*Schema, error) { + s := &Schema{ + schema: schema.New(), + maxParallelism: 10, + tracer: trace.OpenTracingTracer{}, + validationTracer: trace.NoopValidationTracer{}, + logger: &log.DefaultLogger{}, + } + for _, opt := range opts { + opt(s) + } + + if err := s.schema.Parse(schemaString); err != nil { + return nil, err + } + + if resolver != nil { + r, err := resolvable.ApplyResolver(s.schema, resolver) + if err != nil { + return nil, err + } + s.res = r + } + + return s, nil +} + +// MustParseSchema calls ParseSchema and panics on error. +func MustParseSchema(schemaString string, resolver interface{}, opts ...SchemaOpt) *Schema { + s, err := ParseSchema(schemaString, resolver, opts...) + if err != nil { + panic(err) + } + return s +} + +// Schema represents a GraphQL schema with an optional resolver. +type Schema struct { + schema *schema.Schema + res *resolvable.Schema + + maxDepth int + maxParallelism int + tracer trace.Tracer + validationTracer trace.ValidationTracer + logger log.Logger +} + +// SchemaOpt is an option to pass to ParseSchema or MustParseSchema. +type SchemaOpt func(*Schema) + +// MaxDepth specifies the maximum field nesting depth in a query. The default is 0 which disables max depth checking. +func MaxDepth(n int) SchemaOpt { + return func(s *Schema) { + s.maxDepth = n + } +} + +// MaxParallelism specifies the maximum number of resolvers per request allowed to run in parallel. The default is 10. +func MaxParallelism(n int) SchemaOpt { + return func(s *Schema) { + s.maxParallelism = n + } +} + +// Tracer is used to trace queries and fields. It defaults to trace.OpenTracingTracer. +func Tracer(tracer trace.Tracer) SchemaOpt { + return func(s *Schema) { + s.tracer = tracer + } +} + +// ValidationTracer is used to trace validation errors. It defaults to trace.NoopValidationTracer. +func ValidationTracer(tracer trace.ValidationTracer) SchemaOpt { + return func(s *Schema) { + s.validationTracer = tracer + } +} + +// Logger is used to log panics during query execution. It defaults to exec.DefaultLogger. +func Logger(logger log.Logger) SchemaOpt { + return func(s *Schema) { + s.logger = logger + } +} + +// Response represents a typical response of a GraphQL server. It may be encoded to JSON directly or +// it may be further processed to a custom response type, for example to include custom error data. +// Errors are intentionally serialized first based on the advice in https://github.com/facebook/graphql/commit/7b40390d48680b15cb93e02d46ac5eb249689876#diff-757cea6edf0288677a9eea4cfc801d87R107 +type Response struct { + Errors []*errors.QueryError `json:"errors,omitempty"` + Data json.RawMessage `json:"data,omitempty"` + Extensions map[string]interface{} `json:"extensions,omitempty"` +} + +// Validate validates the given query with the schema. +func (s *Schema) Validate(queryString string) []*errors.QueryError { + doc, qErr := query.Parse(queryString) + if qErr != nil { + return []*errors.QueryError{qErr} + } + + return validation.Validate(s.schema, doc, s.maxDepth) +} + +// Exec executes the given query with the schema's resolver. It panics if the schema was created +// without a resolver. If the context get cancelled, no further resolvers will be called and a +// the context error will be returned as soon as possible (not immediately). +func (s *Schema) Exec(ctx context.Context, queryString string, operationName string, variables map[string]interface{}) *Response { + if s.res == nil { + panic("schema created without resolver, can not exec") + } + return s.exec(ctx, queryString, operationName, variables, s.res) +} + +func (s *Schema) exec(ctx context.Context, queryString string, operationName string, variables map[string]interface{}, res *resolvable.Schema) *Response { + doc, qErr := query.Parse(queryString) + if qErr != nil { + return &Response{Errors: []*errors.QueryError{qErr}} + } + + validationFinish := s.validationTracer.TraceValidation() + errs := validation.Validate(s.schema, doc, s.maxDepth) + validationFinish(errs) + if len(errs) != 0 { + return &Response{Errors: errs} + } + + op, err := getOperation(doc, operationName) + if err != nil { + return &Response{Errors: []*errors.QueryError{errors.Errorf("%s", err)}} + } + + r := &exec.Request{ + Request: selected.Request{ + Doc: doc, + Vars: variables, + Schema: s.schema, + }, + Limiter: make(chan struct{}, s.maxParallelism), + Tracer: s.tracer, + Logger: s.logger, + } + varTypes := make(map[string]*introspection.Type) + for _, v := range op.Vars { + t, err := common.ResolveType(v.Type, s.schema.Resolve) + if err != nil { + return &Response{Errors: []*errors.QueryError{err}} + } + varTypes[v.Name.Name] = introspection.WrapType(t) + } + traceCtx, finish := s.tracer.TraceQuery(ctx, queryString, operationName, variables, varTypes) + data, errs := r.Execute(traceCtx, res, op) + finish(errs) + + return &Response{ + Data: data, + Errors: errs, + } +} + +func getOperation(document *query.Document, operationName string) (*query.Operation, error) { + if len(document.Operations) == 0 { + return nil, fmt.Errorf("no operations in query document") + } + + if operationName == "" { + if len(document.Operations) > 1 { + return nil, fmt.Errorf("more than one operation in query document and no operation name given") + } + for _, op := range document.Operations { + return op, nil // return the one and only operation + } + } + + op := document.Operations.Get(operationName) + if op == nil { + return nil, fmt.Errorf("no operation with name %q", operationName) + } + return op, nil +} diff --git a/vendor/github.com/graph-gophers/graphql-go/id.go b/vendor/github.com/graph-gophers/graphql-go/id.go new file mode 100644 index 0000000000..52771c413b --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/id.go @@ -0,0 +1,30 @@ +package graphql + +import ( + "errors" + "strconv" +) + +// ID represents GraphQL's "ID" scalar type. A custom type may be used instead. +type ID string + +func (ID) ImplementsGraphQLType(name string) bool { + return name == "ID" +} + +func (id *ID) UnmarshalGraphQL(input interface{}) error { + var err error + switch input := input.(type) { + case string: + *id = ID(input) + case int32: + *id = ID(strconv.Itoa(int(input))) + default: + err = errors.New("wrong type") + } + return err +} + +func (id ID) MarshalJSON() ([]byte, error) { + return strconv.AppendQuote(nil, string(id)), nil +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/common/directive.go b/vendor/github.com/graph-gophers/graphql-go/internal/common/directive.go new file mode 100644 index 0000000000..62dca47f81 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/common/directive.go @@ -0,0 +1,32 @@ +package common + +type Directive struct { + Name Ident + Args ArgumentList +} + +func ParseDirectives(l *Lexer) DirectiveList { + var directives DirectiveList + for l.Peek() == '@' { + l.ConsumeToken('@') + d := &Directive{} + d.Name = l.ConsumeIdentWithLoc() + d.Name.Loc.Column-- + if l.Peek() == '(' { + d.Args = ParseArguments(l) + } + directives = append(directives, d) + } + return directives +} + +type DirectiveList []*Directive + +func (l DirectiveList) Get(name string) *Directive { + for _, d := range l { + if d.Name.Name == name { + return d + } + } + return nil +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/common/lexer.go b/vendor/github.com/graph-gophers/graphql-go/internal/common/lexer.go new file mode 100644 index 0000000000..a38fcbaf70 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/common/lexer.go @@ -0,0 +1,161 @@ +package common + +import ( + "fmt" + "strings" + "text/scanner" + + "github.com/graph-gophers/graphql-go/errors" +) + +type syntaxError string + +type Lexer struct { + sc *scanner.Scanner + next rune + descComment string +} + +type Ident struct { + Name string + Loc errors.Location +} + +func NewLexer(s string) *Lexer { + sc := &scanner.Scanner{ + Mode: scanner.ScanIdents | scanner.ScanInts | scanner.ScanFloats | scanner.ScanStrings, + } + sc.Init(strings.NewReader(s)) + + return &Lexer{sc: sc} +} + +func (l *Lexer) CatchSyntaxError(f func()) (errRes *errors.QueryError) { + defer func() { + if err := recover(); err != nil { + if err, ok := err.(syntaxError); ok { + errRes = errors.Errorf("syntax error: %s", err) + errRes.Locations = []errors.Location{l.Location()} + return + } + panic(err) + } + }() + + f() + return +} + +func (l *Lexer) Peek() rune { + return l.next +} + +// Consume whitespace and tokens equivalent to whitespace (e.g. commas and comments). +// +// Consumed comment characters will build the description for the next type or field encountered. +// The description is available from `DescComment()`, and will be reset every time `Consume()` is +// executed. +func (l *Lexer) Consume() { + l.descComment = "" + for { + l.next = l.sc.Scan() + + if l.next == ',' { + // Similar to white space and line terminators, commas (',') are used to improve the + // legibility of source text and separate lexical tokens but are otherwise syntactically and + // semantically insignificant within GraphQL documents. + // + // http://facebook.github.io/graphql/draft/#sec-Insignificant-Commas + continue + } + + if l.next == '#' { + // GraphQL source documents may contain single-line comments, starting with the '#' marker. + // + // A comment can contain any Unicode code point except `LineTerminator` so a comment always + // consists of all code points starting with the '#' character up to but not including the + // line terminator. + + l.consumeComment() + continue + } + + break + } +} + +func (l *Lexer) ConsumeIdent() string { + name := l.sc.TokenText() + l.ConsumeToken(scanner.Ident) + return name +} + +func (l *Lexer) ConsumeIdentWithLoc() Ident { + loc := l.Location() + name := l.sc.TokenText() + l.ConsumeToken(scanner.Ident) + return Ident{name, loc} +} + +func (l *Lexer) ConsumeKeyword(keyword string) { + if l.next != scanner.Ident || l.sc.TokenText() != keyword { + l.SyntaxError(fmt.Sprintf("unexpected %q, expecting %q", l.sc.TokenText(), keyword)) + } + l.Consume() +} + +func (l *Lexer) ConsumeLiteral() *BasicLit { + lit := &BasicLit{Type: l.next, Text: l.sc.TokenText()} + l.Consume() + return lit +} + +func (l *Lexer) ConsumeToken(expected rune) { + if l.next != expected { + l.SyntaxError(fmt.Sprintf("unexpected %q, expecting %s", l.sc.TokenText(), scanner.TokenString(expected))) + } + l.Consume() +} + +func (l *Lexer) DescComment() string { + return l.descComment +} + +func (l *Lexer) SyntaxError(message string) { + panic(syntaxError(message)) +} + +func (l *Lexer) Location() errors.Location { + return errors.Location{ + Line: l.sc.Line, + Column: l.sc.Column, + } +} + +// consumeComment consumes all characters from `#` to the first encountered line terminator. +// The characters are appended to `l.descComment`. +func (l *Lexer) consumeComment() { + if l.next != '#' { + return + } + + // TODO: count and trim whitespace so we can dedent any following lines. + if l.sc.Peek() == ' ' { + l.sc.Next() + } + + if l.descComment != "" { + // TODO: use a bytes.Buffer or strings.Builder instead of this. + l.descComment += "\n" + } + + for { + next := l.sc.Next() + if next == '\r' || next == '\n' || next == scanner.EOF { + break + } + + // TODO: use a bytes.Buffer or strings.Build instead of this. + l.descComment += string(next) + } +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/common/literals.go b/vendor/github.com/graph-gophers/graphql-go/internal/common/literals.go new file mode 100644 index 0000000000..e7bbe2638e --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/common/literals.go @@ -0,0 +1,206 @@ +package common + +import ( + "strconv" + "strings" + "text/scanner" + + "github.com/graph-gophers/graphql-go/errors" +) + +type Literal interface { + Value(vars map[string]interface{}) interface{} + String() string + Location() errors.Location +} + +type BasicLit struct { + Type rune + Text string + Loc errors.Location +} + +func (lit *BasicLit) Value(vars map[string]interface{}) interface{} { + switch lit.Type { + case scanner.Int: + value, err := strconv.ParseInt(lit.Text, 10, 32) + if err != nil { + panic(err) + } + return int32(value) + + case scanner.Float: + value, err := strconv.ParseFloat(lit.Text, 64) + if err != nil { + panic(err) + } + return value + + case scanner.String: + value, err := strconv.Unquote(lit.Text) + if err != nil { + panic(err) + } + return value + + case scanner.Ident: + switch lit.Text { + case "true": + return true + case "false": + return false + default: + return lit.Text + } + + default: + panic("invalid literal") + } +} + +func (lit *BasicLit) String() string { + return lit.Text +} + +func (lit *BasicLit) Location() errors.Location { + return lit.Loc +} + +type ListLit struct { + Entries []Literal + Loc errors.Location +} + +func (lit *ListLit) Value(vars map[string]interface{}) interface{} { + entries := make([]interface{}, len(lit.Entries)) + for i, entry := range lit.Entries { + entries[i] = entry.Value(vars) + } + return entries +} + +func (lit *ListLit) String() string { + entries := make([]string, len(lit.Entries)) + for i, entry := range lit.Entries { + entries[i] = entry.String() + } + return "[" + strings.Join(entries, ", ") + "]" +} + +func (lit *ListLit) Location() errors.Location { + return lit.Loc +} + +type ObjectLit struct { + Fields []*ObjectLitField + Loc errors.Location +} + +type ObjectLitField struct { + Name Ident + Value Literal +} + +func (lit *ObjectLit) Value(vars map[string]interface{}) interface{} { + fields := make(map[string]interface{}, len(lit.Fields)) + for _, f := range lit.Fields { + fields[f.Name.Name] = f.Value.Value(vars) + } + return fields +} + +func (lit *ObjectLit) String() string { + entries := make([]string, 0, len(lit.Fields)) + for _, f := range lit.Fields { + entries = append(entries, f.Name.Name+": "+f.Value.String()) + } + return "{" + strings.Join(entries, ", ") + "}" +} + +func (lit *ObjectLit) Location() errors.Location { + return lit.Loc +} + +type NullLit struct { + Loc errors.Location +} + +func (lit *NullLit) Value(vars map[string]interface{}) interface{} { + return nil +} + +func (lit *NullLit) String() string { + return "null" +} + +func (lit *NullLit) Location() errors.Location { + return lit.Loc +} + +type Variable struct { + Name string + Loc errors.Location +} + +func (v Variable) Value(vars map[string]interface{}) interface{} { + return vars[v.Name] +} + +func (v Variable) String() string { + return "$" + v.Name +} + +func (v *Variable) Location() errors.Location { + return v.Loc +} + +func ParseLiteral(l *Lexer, constOnly bool) Literal { + loc := l.Location() + switch l.Peek() { + case '$': + if constOnly { + l.SyntaxError("variable not allowed") + panic("unreachable") + } + l.ConsumeToken('$') + return &Variable{l.ConsumeIdent(), loc} + + case scanner.Int, scanner.Float, scanner.String, scanner.Ident: + lit := l.ConsumeLiteral() + if lit.Type == scanner.Ident && lit.Text == "null" { + return &NullLit{loc} + } + lit.Loc = loc + return lit + case '-': + l.ConsumeToken('-') + lit := l.ConsumeLiteral() + lit.Text = "-" + lit.Text + lit.Loc = loc + return lit + case '[': + l.ConsumeToken('[') + var list []Literal + for l.Peek() != ']' { + list = append(list, ParseLiteral(l, constOnly)) + } + l.ConsumeToken(']') + return &ListLit{list, loc} + + case '{': + l.ConsumeToken('{') + var fields []*ObjectLitField + for l.Peek() != '}' { + name := l.ConsumeIdentWithLoc() + l.ConsumeToken(':') + value := ParseLiteral(l, constOnly) + fields = append(fields, &ObjectLitField{name, value}) + } + l.ConsumeToken('}') + return &ObjectLit{fields, loc} + + default: + l.SyntaxError("invalid value") + panic("unreachable") + } +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/common/types.go b/vendor/github.com/graph-gophers/graphql-go/internal/common/types.go new file mode 100644 index 0000000000..a20ca30906 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/common/types.go @@ -0,0 +1,80 @@ +package common + +import ( + "github.com/graph-gophers/graphql-go/errors" +) + +type Type interface { + Kind() string + String() string +} + +type List struct { + OfType Type +} + +type NonNull struct { + OfType Type +} + +type TypeName struct { + Ident +} + +func (*List) Kind() string { return "LIST" } +func (*NonNull) Kind() string { return "NON_NULL" } +func (*TypeName) Kind() string { panic("TypeName needs to be resolved to actual type") } + +func (t *List) String() string { return "[" + t.OfType.String() + "]" } +func (t *NonNull) String() string { return t.OfType.String() + "!" } +func (*TypeName) String() string { panic("TypeName needs to be resolved to actual type") } + +func ParseType(l *Lexer) Type { + t := parseNullType(l) + if l.Peek() == '!' { + l.ConsumeToken('!') + return &NonNull{OfType: t} + } + return t +} + +func parseNullType(l *Lexer) Type { + if l.Peek() == '[' { + l.ConsumeToken('[') + ofType := ParseType(l) + l.ConsumeToken(']') + return &List{OfType: ofType} + } + + return &TypeName{Ident: l.ConsumeIdentWithLoc()} +} + +type Resolver func(name string) Type + +func ResolveType(t Type, resolver Resolver) (Type, *errors.QueryError) { + switch t := t.(type) { + case *List: + ofType, err := ResolveType(t.OfType, resolver) + if err != nil { + return nil, err + } + return &List{OfType: ofType}, nil + case *NonNull: + ofType, err := ResolveType(t.OfType, resolver) + if err != nil { + return nil, err + } + return &NonNull{OfType: ofType}, nil + case *TypeName: + refT := resolver(t.Name) + if refT == nil { + err := errors.Errorf("Unknown type %q.", t.Name) + err.Rule = "KnownTypeNames" + err.Locations = []errors.Location{t.Loc} + return nil, err + } + return refT, nil + default: + return t, nil + } +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/common/values.go b/vendor/github.com/graph-gophers/graphql-go/internal/common/values.go new file mode 100644 index 0000000000..fcd456abfa --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/common/values.go @@ -0,0 +1,78 @@ +package common + +import ( + "github.com/graph-gophers/graphql-go/errors" +) + +// http://facebook.github.io/graphql/draft/#InputValueDefinition +type InputValue struct { + Name Ident + Type Type + Default Literal + Desc string + Loc errors.Location + TypeLoc errors.Location +} + +type InputValueList []*InputValue + +func (l InputValueList) Get(name string) *InputValue { + for _, v := range l { + if v.Name.Name == name { + return v + } + } + return nil +} + +func ParseInputValue(l *Lexer) *InputValue { + p := &InputValue{} + p.Loc = l.Location() + p.Desc = l.DescComment() + p.Name = l.ConsumeIdentWithLoc() + l.ConsumeToken(':') + p.TypeLoc = l.Location() + p.Type = ParseType(l) + if l.Peek() == '=' { + l.ConsumeToken('=') + p.Default = ParseLiteral(l, true) + } + return p +} + +type Argument struct { + Name Ident + Value Literal +} + +type ArgumentList []Argument + +func (l ArgumentList) Get(name string) (Literal, bool) { + for _, arg := range l { + if arg.Name.Name == name { + return arg.Value, true + } + } + return nil, false +} + +func (l ArgumentList) MustGet(name string) Literal { + value, ok := l.Get(name) + if !ok { + panic("argument not found") + } + return value +} + +func ParseArguments(l *Lexer) ArgumentList { + var args ArgumentList + l.ConsumeToken('(') + for l.Peek() != ')' { + name := l.ConsumeIdentWithLoc() + l.ConsumeToken(':') + value := ParseLiteral(l, false) + args = append(args, Argument{Name: name, Value: value}) + } + l.ConsumeToken(')') + return args +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/exec/exec.go b/vendor/github.com/graph-gophers/graphql-go/internal/exec/exec.go new file mode 100644 index 0000000000..e6cca7448d --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/exec/exec.go @@ -0,0 +1,305 @@ +package exec + +import ( + "bytes" + "context" + "encoding/json" + "reflect" + "sync" + + "github.com/graph-gophers/graphql-go/errors" + "github.com/graph-gophers/graphql-go/internal/common" + "github.com/graph-gophers/graphql-go/internal/exec/resolvable" + "github.com/graph-gophers/graphql-go/internal/exec/selected" + "github.com/graph-gophers/graphql-go/internal/query" + "github.com/graph-gophers/graphql-go/internal/schema" + "github.com/graph-gophers/graphql-go/log" + "github.com/graph-gophers/graphql-go/trace" +) + +type Request struct { + selected.Request + Limiter chan struct{} + Tracer trace.Tracer + Logger log.Logger +} + +func (r *Request) handlePanic(ctx context.Context) { + if value := recover(); value != nil { + r.Logger.LogPanic(ctx, value) + r.AddError(makePanicError(value)) + } +} + +func makePanicError(value interface{}) *errors.QueryError { + return errors.Errorf("graphql: panic occurred: %v", value) +} + +func (r *Request) Execute(ctx context.Context, s *resolvable.Schema, op *query.Operation) ([]byte, []*errors.QueryError) { + var out bytes.Buffer + func() { + defer r.handlePanic(ctx) + sels := selected.ApplyOperation(&r.Request, s, op) + r.execSelections(ctx, sels, nil, s.Resolver, &out, op.Type == query.Mutation) + }() + + if err := ctx.Err(); err != nil { + return nil, []*errors.QueryError{errors.Errorf("%s", err)} + } + + return out.Bytes(), r.Errs +} + +type fieldToExec struct { + field *selected.SchemaField + sels []selected.Selection + resolver reflect.Value + out *bytes.Buffer +} + +func (r *Request) execSelections(ctx context.Context, sels []selected.Selection, path *pathSegment, resolver reflect.Value, out *bytes.Buffer, serially bool) { + async := !serially && selected.HasAsyncSel(sels) + + var fields []*fieldToExec + collectFieldsToResolve(sels, resolver, &fields, make(map[string]*fieldToExec)) + + if async { + var wg sync.WaitGroup + wg.Add(len(fields)) + for _, f := range fields { + go func(f *fieldToExec) { + defer wg.Done() + defer r.handlePanic(ctx) + f.out = new(bytes.Buffer) + execFieldSelection(ctx, r, f, &pathSegment{path, f.field.Alias}, true) + }(f) + } + wg.Wait() + } + + out.WriteByte('{') + for i, f := range fields { + if i > 0 { + out.WriteByte(',') + } + out.WriteByte('"') + out.WriteString(f.field.Alias) + out.WriteByte('"') + out.WriteByte(':') + if async { + out.Write(f.out.Bytes()) + continue + } + f.out = out + execFieldSelection(ctx, r, f, &pathSegment{path, f.field.Alias}, false) + } + out.WriteByte('}') +} + +func collectFieldsToResolve(sels []selected.Selection, resolver reflect.Value, fields *[]*fieldToExec, fieldByAlias map[string]*fieldToExec) { + for _, sel := range sels { + switch sel := sel.(type) { + case *selected.SchemaField: + field, ok := fieldByAlias[sel.Alias] + if !ok { // validation already checked for conflict (TODO) + field = &fieldToExec{field: sel, resolver: resolver} + fieldByAlias[sel.Alias] = field + *fields = append(*fields, field) + } + field.sels = append(field.sels, sel.Sels...) + + case *selected.TypenameField: + sf := &selected.SchemaField{ + Field: resolvable.MetaFieldTypename, + Alias: sel.Alias, + FixedResult: reflect.ValueOf(typeOf(sel, resolver)), + } + *fields = append(*fields, &fieldToExec{field: sf, resolver: resolver}) + + case *selected.TypeAssertion: + out := resolver.Method(sel.MethodIndex).Call(nil) + if !out[1].Bool() { + continue + } + collectFieldsToResolve(sel.Sels, out[0], fields, fieldByAlias) + + default: + panic("unreachable") + } + } +} + +func typeOf(tf *selected.TypenameField, resolver reflect.Value) string { + if len(tf.TypeAssertions) == 0 { + return tf.Name + } + for name, a := range tf.TypeAssertions { + out := resolver.Method(a.MethodIndex).Call(nil) + if out[1].Bool() { + return name + } + } + return "" +} + +func execFieldSelection(ctx context.Context, r *Request, f *fieldToExec, path *pathSegment, applyLimiter bool) { + if applyLimiter { + r.Limiter <- struct{}{} + } + + var result reflect.Value + var err *errors.QueryError + + traceCtx, finish := r.Tracer.TraceField(ctx, f.field.TraceLabel, f.field.TypeName, f.field.Name, !f.field.Async, f.field.Args) + defer func() { + finish(err) + }() + + err = func() (err *errors.QueryError) { + defer func() { + if panicValue := recover(); panicValue != nil { + r.Logger.LogPanic(ctx, panicValue) + err = makePanicError(panicValue) + err.Path = path.toSlice() + } + }() + + if f.field.FixedResult.IsValid() { + result = f.field.FixedResult + return nil + } + + if err := traceCtx.Err(); err != nil { + return errors.Errorf("%s", err) // don't execute any more resolvers if context got cancelled + } + + var in []reflect.Value + if f.field.HasContext { + in = append(in, reflect.ValueOf(traceCtx)) + } + if f.field.ArgsPacker != nil { + in = append(in, f.field.PackedArgs) + } + callOut := f.resolver.Method(f.field.MethodIndex).Call(in) + result = callOut[0] + if f.field.HasError && !callOut[1].IsNil() { + resolverErr := callOut[1].Interface().(error) + err := errors.Errorf("%s", resolverErr) + err.Path = path.toSlice() + err.ResolverError = resolverErr + return err + } + return nil + }() + + if applyLimiter { + <-r.Limiter + } + + if err != nil { + r.AddError(err) + f.out.WriteString("null") // TODO handle non-nil + return + } + + r.execSelectionSet(traceCtx, f.sels, f.field.Type, path, result, f.out) +} + +func (r *Request) execSelectionSet(ctx context.Context, sels []selected.Selection, typ common.Type, path *pathSegment, resolver reflect.Value, out *bytes.Buffer) { + t, nonNull := unwrapNonNull(typ) + switch t := t.(type) { + case *schema.Object, *schema.Interface, *schema.Union: + // a reflect.Value of a nil interface will show up as an Invalid value + if resolver.Kind() == reflect.Invalid || ((resolver.Kind() == reflect.Ptr || resolver.Kind() == reflect.Interface) && resolver.IsNil()) { + if nonNull { + panic(errors.Errorf("got nil for non-null %q", t)) + } + out.WriteString("null") + return + } + + r.execSelections(ctx, sels, path, resolver, out, false) + return + } + + if !nonNull { + if resolver.IsNil() { + out.WriteString("null") + return + } + resolver = resolver.Elem() + } + + switch t := t.(type) { + case *common.List: + l := resolver.Len() + + if selected.HasAsyncSel(sels) { + var wg sync.WaitGroup + wg.Add(l) + entryouts := make([]bytes.Buffer, l) + for i := 0; i < l; i++ { + go func(i int) { + defer wg.Done() + defer r.handlePanic(ctx) + r.execSelectionSet(ctx, sels, t.OfType, &pathSegment{path, i}, resolver.Index(i), &entryouts[i]) + }(i) + } + wg.Wait() + + out.WriteByte('[') + for i, entryout := range entryouts { + if i > 0 { + out.WriteByte(',') + } + out.Write(entryout.Bytes()) + } + out.WriteByte(']') + return + } + + out.WriteByte('[') + for i := 0; i < l; i++ { + if i > 0 { + out.WriteByte(',') + } + r.execSelectionSet(ctx, sels, t.OfType, &pathSegment{path, i}, resolver.Index(i), out) + } + out.WriteByte(']') + + case *schema.Scalar: + v := resolver.Interface() + data, err := json.Marshal(v) + if err != nil { + panic(errors.Errorf("could not marshal %v: %s", v, err)) + } + out.Write(data) + + case *schema.Enum: + out.WriteByte('"') + out.WriteString(resolver.String()) + out.WriteByte('"') + + default: + panic("unreachable") + } +} + +func unwrapNonNull(t common.Type) (common.Type, bool) { + if nn, ok := t.(*common.NonNull); ok { + return nn.OfType, true + } + return t, false +} + +type pathSegment struct { + parent *pathSegment + value interface{} +} + +func (p *pathSegment) toSlice() []interface{} { + if p == nil { + return nil + } + return append(p.parent.toSlice(), p.value) +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/exec/packer/packer.go b/vendor/github.com/graph-gophers/graphql-go/internal/exec/packer/packer.go new file mode 100644 index 0000000000..22706bcd1f --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/exec/packer/packer.go @@ -0,0 +1,371 @@ +package packer + +import ( + "fmt" + "math" + "reflect" + "strings" + + "github.com/graph-gophers/graphql-go/errors" + "github.com/graph-gophers/graphql-go/internal/common" + "github.com/graph-gophers/graphql-go/internal/schema" +) + +type packer interface { + Pack(value interface{}) (reflect.Value, error) +} + +type Builder struct { + packerMap map[typePair]*packerMapEntry + structPackers []*StructPacker +} + +type typePair struct { + graphQLType common.Type + resolverType reflect.Type +} + +type packerMapEntry struct { + packer packer + targets []*packer +} + +func NewBuilder() *Builder { + return &Builder{ + packerMap: make(map[typePair]*packerMapEntry), + } +} + +func (b *Builder) Finish() error { + for _, entry := range b.packerMap { + for _, target := range entry.targets { + *target = entry.packer + } + } + + for _, p := range b.structPackers { + p.defaultStruct = reflect.New(p.structType).Elem() + for _, f := range p.fields { + if defaultVal := f.field.Default; defaultVal != nil { + v, err := f.fieldPacker.Pack(defaultVal.Value(nil)) + if err != nil { + return err + } + p.defaultStruct.FieldByIndex(f.fieldIndex).Set(v) + } + } + } + + return nil +} + +func (b *Builder) assignPacker(target *packer, schemaType common.Type, reflectType reflect.Type) error { + k := typePair{schemaType, reflectType} + ref, ok := b.packerMap[k] + if !ok { + ref = &packerMapEntry{} + b.packerMap[k] = ref + var err error + ref.packer, err = b.makePacker(schemaType, reflectType) + if err != nil { + return err + } + } + ref.targets = append(ref.targets, target) + return nil +} + +func (b *Builder) makePacker(schemaType common.Type, reflectType reflect.Type) (packer, error) { + t, nonNull := unwrapNonNull(schemaType) + if !nonNull { + if reflectType.Kind() != reflect.Ptr { + return nil, fmt.Errorf("%s is not a pointer", reflectType) + } + elemType := reflectType.Elem() + addPtr := true + if _, ok := t.(*schema.InputObject); ok { + elemType = reflectType // keep pointer for input objects + addPtr = false + } + elem, err := b.makeNonNullPacker(t, elemType) + if err != nil { + return nil, err + } + return &nullPacker{ + elemPacker: elem, + valueType: reflectType, + addPtr: addPtr, + }, nil + } + + return b.makeNonNullPacker(t, reflectType) +} + +func (b *Builder) makeNonNullPacker(schemaType common.Type, reflectType reflect.Type) (packer, error) { + if u, ok := reflect.New(reflectType).Interface().(Unmarshaler); ok { + if !u.ImplementsGraphQLType(schemaType.String()) { + return nil, fmt.Errorf("can not unmarshal %s into %s", schemaType, reflectType) + } + return &unmarshalerPacker{ + ValueType: reflectType, + }, nil + } + + switch t := schemaType.(type) { + case *schema.Scalar: + return &ValuePacker{ + ValueType: reflectType, + }, nil + + case *schema.Enum: + if reflectType.Kind() != reflect.String { + return nil, fmt.Errorf("wrong type, expected %s", reflect.String) + } + return &ValuePacker{ + ValueType: reflectType, + }, nil + + case *schema.InputObject: + e, err := b.MakeStructPacker(t.Values, reflectType) + if err != nil { + return nil, err + } + return e, nil + + case *common.List: + if reflectType.Kind() != reflect.Slice { + return nil, fmt.Errorf("expected slice, got %s", reflectType) + } + p := &listPacker{ + sliceType: reflectType, + } + if err := b.assignPacker(&p.elem, t.OfType, reflectType.Elem()); err != nil { + return nil, err + } + return p, nil + + case *schema.Object, *schema.Interface, *schema.Union: + return nil, fmt.Errorf("type of kind %s can not be used as input", t.Kind()) + + default: + panic("unreachable") + } +} + +func (b *Builder) MakeStructPacker(values common.InputValueList, typ reflect.Type) (*StructPacker, error) { + structType := typ + usePtr := false + if typ.Kind() == reflect.Ptr { + structType = typ.Elem() + usePtr = true + } + if structType.Kind() != reflect.Struct { + return nil, fmt.Errorf("expected struct or pointer to struct, got %s", typ) + } + + var fields []*structPackerField + for _, v := range values { + fe := &structPackerField{field: v} + fx := func(n string) bool { + return strings.EqualFold(stripUnderscore(n), stripUnderscore(v.Name.Name)) + } + + sf, ok := structType.FieldByNameFunc(fx) + if !ok { + return nil, fmt.Errorf("missing argument %q", v.Name) + } + if sf.PkgPath != "" { + return nil, fmt.Errorf("field %q must be exported", sf.Name) + } + fe.fieldIndex = sf.Index + + ft := v.Type + if v.Default != nil { + ft, _ = unwrapNonNull(ft) + ft = &common.NonNull{OfType: ft} + } + + if err := b.assignPacker(&fe.fieldPacker, ft, sf.Type); err != nil { + return nil, fmt.Errorf("field %q: %s", sf.Name, err) + } + + fields = append(fields, fe) + } + + p := &StructPacker{ + structType: structType, + usePtr: usePtr, + fields: fields, + } + b.structPackers = append(b.structPackers, p) + return p, nil +} + +type StructPacker struct { + structType reflect.Type + usePtr bool + defaultStruct reflect.Value + fields []*structPackerField +} + +type structPackerField struct { + field *common.InputValue + fieldIndex []int + fieldPacker packer +} + +func (p *StructPacker) Pack(value interface{}) (reflect.Value, error) { + if value == nil { + return reflect.Value{}, errors.Errorf("got null for non-null") + } + + values := value.(map[string]interface{}) + v := reflect.New(p.structType) + v.Elem().Set(p.defaultStruct) + for _, f := range p.fields { + if value, ok := values[f.field.Name.Name]; ok { + packed, err := f.fieldPacker.Pack(value) + if err != nil { + return reflect.Value{}, err + } + v.Elem().FieldByIndex(f.fieldIndex).Set(packed) + } + } + if !p.usePtr { + return v.Elem(), nil + } + return v, nil +} + +type listPacker struct { + sliceType reflect.Type + elem packer +} + +func (e *listPacker) Pack(value interface{}) (reflect.Value, error) { + list, ok := value.([]interface{}) + if !ok { + list = []interface{}{value} + } + + v := reflect.MakeSlice(e.sliceType, len(list), len(list)) + for i := range list { + packed, err := e.elem.Pack(list[i]) + if err != nil { + return reflect.Value{}, err + } + v.Index(i).Set(packed) + } + return v, nil +} + +type nullPacker struct { + elemPacker packer + valueType reflect.Type + addPtr bool +} + +func (p *nullPacker) Pack(value interface{}) (reflect.Value, error) { + if value == nil { + return reflect.Zero(p.valueType), nil + } + + v, err := p.elemPacker.Pack(value) + if err != nil { + return reflect.Value{}, err + } + + if p.addPtr { + ptr := reflect.New(p.valueType.Elem()) + ptr.Elem().Set(v) + return ptr, nil + } + + return v, nil +} + +type ValuePacker struct { + ValueType reflect.Type +} + +func (p *ValuePacker) Pack(value interface{}) (reflect.Value, error) { + if value == nil { + return reflect.Value{}, errors.Errorf("got null for non-null") + } + + coerced, err := unmarshalInput(p.ValueType, value) + if err != nil { + return reflect.Value{}, fmt.Errorf("could not unmarshal %#v (%T) into %s: %s", value, value, p.ValueType, err) + } + return reflect.ValueOf(coerced), nil +} + +type unmarshalerPacker struct { + ValueType reflect.Type +} + +func (p *unmarshalerPacker) Pack(value interface{}) (reflect.Value, error) { + if value == nil { + return reflect.Value{}, errors.Errorf("got null for non-null") + } + + v := reflect.New(p.ValueType) + if err := v.Interface().(Unmarshaler).UnmarshalGraphQL(value); err != nil { + return reflect.Value{}, err + } + return v.Elem(), nil +} + +type Unmarshaler interface { + ImplementsGraphQLType(name string) bool + UnmarshalGraphQL(input interface{}) error +} + +func unmarshalInput(typ reflect.Type, input interface{}) (interface{}, error) { + if reflect.TypeOf(input) == typ { + return input, nil + } + + switch typ.Kind() { + case reflect.Int32: + switch input := input.(type) { + case int: + if input < math.MinInt32 || input > math.MaxInt32 { + return nil, fmt.Errorf("not a 32-bit integer") + } + return int32(input), nil + case float64: + coerced := int32(input) + if input < math.MinInt32 || input > math.MaxInt32 || float64(coerced) != input { + return nil, fmt.Errorf("not a 32-bit integer") + } + return coerced, nil + } + + case reflect.Float64: + switch input := input.(type) { + case int32: + return float64(input), nil + case int: + return float64(input), nil + } + + case reflect.String: + if reflect.TypeOf(input).ConvertibleTo(typ) { + return reflect.ValueOf(input).Convert(typ).Interface(), nil + } + } + + return nil, fmt.Errorf("incompatible type") +} + +func unwrapNonNull(t common.Type) (common.Type, bool) { + if nn, ok := t.(*common.NonNull); ok { + return nn.OfType, true + } + return t, false +} + +func stripUnderscore(s string) string { + return strings.Replace(s, "_", "", -1) +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/exec/resolvable/meta.go b/vendor/github.com/graph-gophers/graphql-go/internal/exec/resolvable/meta.go new file mode 100644 index 0000000000..826c823484 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/exec/resolvable/meta.go @@ -0,0 +1,58 @@ +package resolvable + +import ( + "fmt" + "reflect" + + "github.com/graph-gophers/graphql-go/internal/common" + "github.com/graph-gophers/graphql-go/internal/schema" + "github.com/graph-gophers/graphql-go/introspection" +) + +var MetaSchema *Object +var MetaType *Object + +func init() { + var err error + b := newBuilder(schema.Meta) + + metaSchema := schema.Meta.Types["__Schema"].(*schema.Object) + MetaSchema, err = b.makeObjectExec(metaSchema.Name, metaSchema.Fields, nil, false, reflect.TypeOf(&introspection.Schema{})) + if err != nil { + panic(err) + } + + metaType := schema.Meta.Types["__Type"].(*schema.Object) + MetaType, err = b.makeObjectExec(metaType.Name, metaType.Fields, nil, false, reflect.TypeOf(&introspection.Type{})) + if err != nil { + panic(err) + } + + if err := b.finish(); err != nil { + panic(err) + } +} + +var MetaFieldTypename = Field{ + Field: schema.Field{ + Name: "__typename", + Type: &common.NonNull{OfType: schema.Meta.Types["String"]}, + }, + TraceLabel: fmt.Sprintf("GraphQL field: __typename"), +} + +var MetaFieldSchema = Field{ + Field: schema.Field{ + Name: "__schema", + Type: schema.Meta.Types["__Schema"], + }, + TraceLabel: fmt.Sprintf("GraphQL field: __schema"), +} + +var MetaFieldType = Field{ + Field: schema.Field{ + Name: "__type", + Type: schema.Meta.Types["__Type"], + }, + TraceLabel: fmt.Sprintf("GraphQL field: __type"), +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/exec/resolvable/resolvable.go b/vendor/github.com/graph-gophers/graphql-go/internal/exec/resolvable/resolvable.go new file mode 100644 index 0000000000..3e5d9e44d9 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/exec/resolvable/resolvable.go @@ -0,0 +1,331 @@ +package resolvable + +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/graph-gophers/graphql-go/internal/common" + "github.com/graph-gophers/graphql-go/internal/exec/packer" + "github.com/graph-gophers/graphql-go/internal/schema" +) + +type Schema struct { + schema.Schema + Query Resolvable + Mutation Resolvable + Resolver reflect.Value +} + +type Resolvable interface { + isResolvable() +} + +type Object struct { + Name string + Fields map[string]*Field + TypeAssertions map[string]*TypeAssertion +} + +type Field struct { + schema.Field + TypeName string + MethodIndex int + HasContext bool + HasError bool + ArgsPacker *packer.StructPacker + ValueExec Resolvable + TraceLabel string +} + +type TypeAssertion struct { + MethodIndex int + TypeExec Resolvable +} + +type List struct { + Elem Resolvable +} + +type Scalar struct{} + +func (*Object) isResolvable() {} +func (*List) isResolvable() {} +func (*Scalar) isResolvable() {} + +func ApplyResolver(s *schema.Schema, resolver interface{}) (*Schema, error) { + b := newBuilder(s) + + var query, mutation Resolvable + + if t, ok := s.EntryPoints["query"]; ok { + if err := b.assignExec(&query, t, reflect.TypeOf(resolver)); err != nil { + return nil, err + } + } + + if t, ok := s.EntryPoints["mutation"]; ok { + if err := b.assignExec(&mutation, t, reflect.TypeOf(resolver)); err != nil { + return nil, err + } + } + + if err := b.finish(); err != nil { + return nil, err + } + + return &Schema{ + Schema: *s, + Resolver: reflect.ValueOf(resolver), + Query: query, + Mutation: mutation, + }, nil +} + +type execBuilder struct { + schema *schema.Schema + resMap map[typePair]*resMapEntry + packerBuilder *packer.Builder +} + +type typePair struct { + graphQLType common.Type + resolverType reflect.Type +} + +type resMapEntry struct { + exec Resolvable + targets []*Resolvable +} + +func newBuilder(s *schema.Schema) *execBuilder { + return &execBuilder{ + schema: s, + resMap: make(map[typePair]*resMapEntry), + packerBuilder: packer.NewBuilder(), + } +} + +func (b *execBuilder) finish() error { + for _, entry := range b.resMap { + for _, target := range entry.targets { + *target = entry.exec + } + } + + return b.packerBuilder.Finish() +} + +func (b *execBuilder) assignExec(target *Resolvable, t common.Type, resolverType reflect.Type) error { + k := typePair{t, resolverType} + ref, ok := b.resMap[k] + if !ok { + ref = &resMapEntry{} + b.resMap[k] = ref + var err error + ref.exec, err = b.makeExec(t, resolverType) + if err != nil { + return err + } + } + ref.targets = append(ref.targets, target) + return nil +} + +func (b *execBuilder) makeExec(t common.Type, resolverType reflect.Type) (Resolvable, error) { + var nonNull bool + t, nonNull = unwrapNonNull(t) + + switch t := t.(type) { + case *schema.Object: + return b.makeObjectExec(t.Name, t.Fields, nil, nonNull, resolverType) + + case *schema.Interface: + return b.makeObjectExec(t.Name, t.Fields, t.PossibleTypes, nonNull, resolverType) + + case *schema.Union: + return b.makeObjectExec(t.Name, nil, t.PossibleTypes, nonNull, resolverType) + } + + if !nonNull { + if resolverType.Kind() != reflect.Ptr { + return nil, fmt.Errorf("%s is not a pointer", resolverType) + } + resolverType = resolverType.Elem() + } + + switch t := t.(type) { + case *schema.Scalar: + return makeScalarExec(t, resolverType) + + case *schema.Enum: + return &Scalar{}, nil + + case *common.List: + if resolverType.Kind() != reflect.Slice { + return nil, fmt.Errorf("%s is not a slice", resolverType) + } + e := &List{} + if err := b.assignExec(&e.Elem, t.OfType, resolverType.Elem()); err != nil { + return nil, err + } + return e, nil + + default: + panic("invalid type: " + t.String()) + } +} + +func makeScalarExec(t *schema.Scalar, resolverType reflect.Type) (Resolvable, error) { + implementsType := false + switch r := reflect.New(resolverType).Interface().(type) { + case *int32: + implementsType = (t.Name == "Int") + case *float64: + implementsType = (t.Name == "Float") + case *string: + implementsType = (t.Name == "String") + case *bool: + implementsType = (t.Name == "Boolean") + case packer.Unmarshaler: + implementsType = r.ImplementsGraphQLType(t.Name) + } + if !implementsType { + return nil, fmt.Errorf("can not use %s as %s", resolverType, t.Name) + } + return &Scalar{}, nil +} + +func (b *execBuilder) makeObjectExec(typeName string, fields schema.FieldList, possibleTypes []*schema.Object, nonNull bool, resolverType reflect.Type) (*Object, error) { + if !nonNull { + if resolverType.Kind() != reflect.Ptr && resolverType.Kind() != reflect.Interface { + return nil, fmt.Errorf("%s is not a pointer or interface", resolverType) + } + } + + methodHasReceiver := resolverType.Kind() != reflect.Interface + + Fields := make(map[string]*Field) + for _, f := range fields { + methodIndex := findMethod(resolverType, f.Name) + if methodIndex == -1 { + hint := "" + if findMethod(reflect.PtrTo(resolverType), f.Name) != -1 { + hint = " (hint: the method exists on the pointer type)" + } + return nil, fmt.Errorf("%s does not resolve %q: missing method for field %q%s", resolverType, typeName, f.Name, hint) + } + + m := resolverType.Method(methodIndex) + fe, err := b.makeFieldExec(typeName, f, m, methodIndex, methodHasReceiver) + if err != nil { + return nil, fmt.Errorf("%s\n\treturned by (%s).%s", err, resolverType, m.Name) + } + Fields[f.Name] = fe + } + + typeAssertions := make(map[string]*TypeAssertion) + for _, impl := range possibleTypes { + methodIndex := findMethod(resolverType, "To"+impl.Name) + if methodIndex == -1 { + return nil, fmt.Errorf("%s does not resolve %q: missing method %q to convert to %q", resolverType, typeName, "To"+impl.Name, impl.Name) + } + if resolverType.Method(methodIndex).Type.NumOut() != 2 { + return nil, fmt.Errorf("%s does not resolve %q: method %q should return a value and a bool indicating success", resolverType, typeName, "To"+impl.Name) + } + a := &TypeAssertion{ + MethodIndex: methodIndex, + } + if err := b.assignExec(&a.TypeExec, impl, resolverType.Method(methodIndex).Type.Out(0)); err != nil { + return nil, err + } + typeAssertions[impl.Name] = a + } + + return &Object{ + Name: typeName, + Fields: Fields, + TypeAssertions: typeAssertions, + }, nil +} + +var contextType = reflect.TypeOf((*context.Context)(nil)).Elem() +var errorType = reflect.TypeOf((*error)(nil)).Elem() + +func (b *execBuilder) makeFieldExec(typeName string, f *schema.Field, m reflect.Method, methodIndex int, methodHasReceiver bool) (*Field, error) { + in := make([]reflect.Type, m.Type.NumIn()) + for i := range in { + in[i] = m.Type.In(i) + } + if methodHasReceiver { + in = in[1:] // first parameter is receiver + } + + hasContext := len(in) > 0 && in[0] == contextType + if hasContext { + in = in[1:] + } + + var argsPacker *packer.StructPacker + if len(f.Args) > 0 { + if len(in) == 0 { + return nil, fmt.Errorf("must have parameter for field arguments") + } + var err error + argsPacker, err = b.packerBuilder.MakeStructPacker(f.Args, in[0]) + if err != nil { + return nil, err + } + in = in[1:] + } + + if len(in) > 0 { + return nil, fmt.Errorf("too many parameters") + } + + if m.Type.NumOut() > 2 { + return nil, fmt.Errorf("too many return values") + } + + hasError := m.Type.NumOut() == 2 + if hasError { + if m.Type.Out(1) != errorType { + return nil, fmt.Errorf(`must have "error" as its second return value`) + } + } + + fe := &Field{ + Field: *f, + TypeName: typeName, + MethodIndex: methodIndex, + HasContext: hasContext, + ArgsPacker: argsPacker, + HasError: hasError, + TraceLabel: fmt.Sprintf("GraphQL field: %s.%s", typeName, f.Name), + } + if err := b.assignExec(&fe.ValueExec, f.Type, m.Type.Out(0)); err != nil { + return nil, err + } + return fe, nil +} + +func findMethod(t reflect.Type, name string) int { + for i := 0; i < t.NumMethod(); i++ { + if strings.EqualFold(stripUnderscore(name), stripUnderscore(t.Method(i).Name)) { + return i + } + } + return -1 +} + +func unwrapNonNull(t common.Type) (common.Type, bool) { + if nn, ok := t.(*common.NonNull); ok { + return nn.OfType, true + } + return t, false +} + +func stripUnderscore(s string) string { + return strings.Replace(s, "_", "", -1) +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/exec/selected/selected.go b/vendor/github.com/graph-gophers/graphql-go/internal/exec/selected/selected.go new file mode 100644 index 0000000000..aed079b671 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/exec/selected/selected.go @@ -0,0 +1,238 @@ +package selected + +import ( + "fmt" + "reflect" + "sync" + + "github.com/graph-gophers/graphql-go/errors" + "github.com/graph-gophers/graphql-go/internal/common" + "github.com/graph-gophers/graphql-go/internal/exec/packer" + "github.com/graph-gophers/graphql-go/internal/exec/resolvable" + "github.com/graph-gophers/graphql-go/internal/query" + "github.com/graph-gophers/graphql-go/internal/schema" + "github.com/graph-gophers/graphql-go/introspection" +) + +type Request struct { + Schema *schema.Schema + Doc *query.Document + Vars map[string]interface{} + Mu sync.Mutex + Errs []*errors.QueryError +} + +func (r *Request) AddError(err *errors.QueryError) { + r.Mu.Lock() + r.Errs = append(r.Errs, err) + r.Mu.Unlock() +} + +func ApplyOperation(r *Request, s *resolvable.Schema, op *query.Operation) []Selection { + var obj *resolvable.Object + switch op.Type { + case query.Query: + obj = s.Query.(*resolvable.Object) + case query.Mutation: + obj = s.Mutation.(*resolvable.Object) + } + return applySelectionSet(r, obj, op.Selections) +} + +type Selection interface { + isSelection() +} + +type SchemaField struct { + resolvable.Field + Alias string + Args map[string]interface{} + PackedArgs reflect.Value + Sels []Selection + Async bool + FixedResult reflect.Value +} + +type TypeAssertion struct { + resolvable.TypeAssertion + Sels []Selection +} + +type TypenameField struct { + resolvable.Object + Alias string +} + +func (*SchemaField) isSelection() {} +func (*TypeAssertion) isSelection() {} +func (*TypenameField) isSelection() {} + +func applySelectionSet(r *Request, e *resolvable.Object, sels []query.Selection) (flattenedSels []Selection) { + for _, sel := range sels { + switch sel := sel.(type) { + case *query.Field: + field := sel + if skipByDirective(r, field.Directives) { + continue + } + + switch field.Name.Name { + case "__typename": + flattenedSels = append(flattenedSels, &TypenameField{ + Object: *e, + Alias: field.Alias.Name, + }) + + case "__schema": + flattenedSels = append(flattenedSels, &SchemaField{ + Field: resolvable.MetaFieldSchema, + Alias: field.Alias.Name, + Sels: applySelectionSet(r, resolvable.MetaSchema, field.Selections), + Async: true, + FixedResult: reflect.ValueOf(introspection.WrapSchema(r.Schema)), + }) + + case "__type": + p := packer.ValuePacker{ValueType: reflect.TypeOf("")} + v, err := p.Pack(field.Arguments.MustGet("name").Value(r.Vars)) + if err != nil { + r.AddError(errors.Errorf("%s", err)) + return nil + } + + t, ok := r.Schema.Types[v.String()] + if !ok { + return nil + } + + flattenedSels = append(flattenedSels, &SchemaField{ + Field: resolvable.MetaFieldType, + Alias: field.Alias.Name, + Sels: applySelectionSet(r, resolvable.MetaType, field.Selections), + Async: true, + FixedResult: reflect.ValueOf(introspection.WrapType(t)), + }) + + default: + fe := e.Fields[field.Name.Name] + + var args map[string]interface{} + var packedArgs reflect.Value + if fe.ArgsPacker != nil { + args = make(map[string]interface{}) + for _, arg := range field.Arguments { + args[arg.Name.Name] = arg.Value.Value(r.Vars) + } + var err error + packedArgs, err = fe.ArgsPacker.Pack(args) + if err != nil { + r.AddError(errors.Errorf("%s", err)) + return + } + } + + fieldSels := applyField(r, fe.ValueExec, field.Selections) + flattenedSels = append(flattenedSels, &SchemaField{ + Field: *fe, + Alias: field.Alias.Name, + Args: args, + PackedArgs: packedArgs, + Sels: fieldSels, + Async: fe.HasContext || fe.ArgsPacker != nil || fe.HasError || HasAsyncSel(fieldSels), + }) + } + + case *query.InlineFragment: + frag := sel + if skipByDirective(r, frag.Directives) { + continue + } + flattenedSels = append(flattenedSels, applyFragment(r, e, &frag.Fragment)...) + + case *query.FragmentSpread: + spread := sel + if skipByDirective(r, spread.Directives) { + continue + } + flattenedSels = append(flattenedSels, applyFragment(r, e, &r.Doc.Fragments.Get(spread.Name.Name).Fragment)...) + + default: + panic("invalid type") + } + } + return +} + +func applyFragment(r *Request, e *resolvable.Object, frag *query.Fragment) []Selection { + if frag.On.Name != "" && frag.On.Name != e.Name { + a, ok := e.TypeAssertions[frag.On.Name] + if !ok { + panic(fmt.Errorf("%q does not implement %q", frag.On, e.Name)) // TODO proper error handling + } + + return []Selection{&TypeAssertion{ + TypeAssertion: *a, + Sels: applySelectionSet(r, a.TypeExec.(*resolvable.Object), frag.Selections), + }} + } + return applySelectionSet(r, e, frag.Selections) +} + +func applyField(r *Request, e resolvable.Resolvable, sels []query.Selection) []Selection { + switch e := e.(type) { + case *resolvable.Object: + return applySelectionSet(r, e, sels) + case *resolvable.List: + return applyField(r, e.Elem, sels) + case *resolvable.Scalar: + return nil + default: + panic("unreachable") + } +} + +func skipByDirective(r *Request, directives common.DirectiveList) bool { + if d := directives.Get("skip"); d != nil { + p := packer.ValuePacker{ValueType: reflect.TypeOf(false)} + v, err := p.Pack(d.Args.MustGet("if").Value(r.Vars)) + if err != nil { + r.AddError(errors.Errorf("%s", err)) + } + if err == nil && v.Bool() { + return true + } + } + + if d := directives.Get("include"); d != nil { + p := packer.ValuePacker{ValueType: reflect.TypeOf(false)} + v, err := p.Pack(d.Args.MustGet("if").Value(r.Vars)) + if err != nil { + r.AddError(errors.Errorf("%s", err)) + } + if err == nil && !v.Bool() { + return true + } + } + + return false +} + +func HasAsyncSel(sels []Selection) bool { + for _, sel := range sels { + switch sel := sel.(type) { + case *SchemaField: + if sel.Async { + return true + } + case *TypeAssertion: + if HasAsyncSel(sel.Sels) { + return true + } + case *TypenameField: + // sync + default: + panic("unreachable") + } + } + return false +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/query/query.go b/vendor/github.com/graph-gophers/graphql-go/internal/query/query.go new file mode 100644 index 0000000000..faba4d2ade --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/query/query.go @@ -0,0 +1,234 @@ +package query + +import ( + "fmt" + "text/scanner" + + "github.com/graph-gophers/graphql-go/errors" + "github.com/graph-gophers/graphql-go/internal/common" +) + +type Document struct { + Operations OperationList + Fragments FragmentList +} + +type OperationList []*Operation + +func (l OperationList) Get(name string) *Operation { + for _, f := range l { + if f.Name.Name == name { + return f + } + } + return nil +} + +type FragmentList []*FragmentDecl + +func (l FragmentList) Get(name string) *FragmentDecl { + for _, f := range l { + if f.Name.Name == name { + return f + } + } + return nil +} + +type Operation struct { + Type OperationType + Name common.Ident + Vars common.InputValueList + Selections []Selection + Directives common.DirectiveList + Loc errors.Location +} + +type OperationType string + +const ( + Query OperationType = "QUERY" + Mutation = "MUTATION" + Subscription = "SUBSCRIPTION" +) + +type Fragment struct { + On common.TypeName + Selections []Selection +} + +type FragmentDecl struct { + Fragment + Name common.Ident + Directives common.DirectiveList + Loc errors.Location +} + +type Selection interface { + isSelection() +} + +type Field struct { + Alias common.Ident + Name common.Ident + Arguments common.ArgumentList + Directives common.DirectiveList + Selections []Selection + SelectionSetLoc errors.Location +} + +type InlineFragment struct { + Fragment + Directives common.DirectiveList + Loc errors.Location +} + +type FragmentSpread struct { + Name common.Ident + Directives common.DirectiveList + Loc errors.Location +} + +func (Field) isSelection() {} +func (InlineFragment) isSelection() {} +func (FragmentSpread) isSelection() {} + +func Parse(queryString string) (*Document, *errors.QueryError) { + l := common.NewLexer(queryString) + + var doc *Document + err := l.CatchSyntaxError(func() { doc = parseDocument(l) }) + if err != nil { + return nil, err + } + + return doc, nil +} + +func parseDocument(l *common.Lexer) *Document { + d := &Document{} + l.Consume() + for l.Peek() != scanner.EOF { + if l.Peek() == '{' { + op := &Operation{Type: Query, Loc: l.Location()} + op.Selections = parseSelectionSet(l) + d.Operations = append(d.Operations, op) + continue + } + + loc := l.Location() + switch x := l.ConsumeIdent(); x { + case "query": + op := parseOperation(l, Query) + op.Loc = loc + d.Operations = append(d.Operations, op) + + case "mutation": + d.Operations = append(d.Operations, parseOperation(l, Mutation)) + + case "subscription": + d.Operations = append(d.Operations, parseOperation(l, Subscription)) + + case "fragment": + frag := parseFragment(l) + frag.Loc = loc + d.Fragments = append(d.Fragments, frag) + + default: + l.SyntaxError(fmt.Sprintf(`unexpected %q, expecting "fragment"`, x)) + } + } + return d +} + +func parseOperation(l *common.Lexer, opType OperationType) *Operation { + op := &Operation{Type: opType} + op.Name.Loc = l.Location() + if l.Peek() == scanner.Ident { + op.Name = l.ConsumeIdentWithLoc() + } + op.Directives = common.ParseDirectives(l) + if l.Peek() == '(' { + l.ConsumeToken('(') + for l.Peek() != ')' { + loc := l.Location() + l.ConsumeToken('$') + iv := common.ParseInputValue(l) + iv.Loc = loc + op.Vars = append(op.Vars, iv) + } + l.ConsumeToken(')') + } + op.Selections = parseSelectionSet(l) + return op +} + +func parseFragment(l *common.Lexer) *FragmentDecl { + f := &FragmentDecl{} + f.Name = l.ConsumeIdentWithLoc() + l.ConsumeKeyword("on") + f.On = common.TypeName{Ident: l.ConsumeIdentWithLoc()} + f.Directives = common.ParseDirectives(l) + f.Selections = parseSelectionSet(l) + return f +} + +func parseSelectionSet(l *common.Lexer) []Selection { + var sels []Selection + l.ConsumeToken('{') + for l.Peek() != '}' { + sels = append(sels, parseSelection(l)) + } + l.ConsumeToken('}') + return sels +} + +func parseSelection(l *common.Lexer) Selection { + if l.Peek() == '.' { + return parseSpread(l) + } + return parseField(l) +} + +func parseField(l *common.Lexer) *Field { + f := &Field{} + f.Alias = l.ConsumeIdentWithLoc() + f.Name = f.Alias + if l.Peek() == ':' { + l.ConsumeToken(':') + f.Name = l.ConsumeIdentWithLoc() + } + if l.Peek() == '(' { + f.Arguments = common.ParseArguments(l) + } + f.Directives = common.ParseDirectives(l) + if l.Peek() == '{' { + f.SelectionSetLoc = l.Location() + f.Selections = parseSelectionSet(l) + } + return f +} + +func parseSpread(l *common.Lexer) Selection { + loc := l.Location() + l.ConsumeToken('.') + l.ConsumeToken('.') + l.ConsumeToken('.') + + f := &InlineFragment{Loc: loc} + if l.Peek() == scanner.Ident { + ident := l.ConsumeIdentWithLoc() + if ident.Name != "on" { + fs := &FragmentSpread{ + Name: ident, + Loc: loc, + } + fs.Directives = common.ParseDirectives(l) + return fs + } + f.On = common.TypeName{Ident: l.ConsumeIdentWithLoc()} + } + f.Directives = common.ParseDirectives(l) + f.Selections = parseSelectionSet(l) + return f +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/schema/meta.go b/vendor/github.com/graph-gophers/graphql-go/internal/schema/meta.go new file mode 100644 index 0000000000..b48bf7acf2 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/schema/meta.go @@ -0,0 +1,190 @@ +package schema + +var Meta *Schema + +func init() { + Meta = &Schema{} // bootstrap + Meta = New() + if err := Meta.Parse(metaSrc); err != nil { + panic(err) + } +} + +var metaSrc = ` + # The ` + "`" + `Int` + "`" + ` scalar type represents non-fractional signed whole numeric values. Int can represent values between -(2^31) and 2^31 - 1. + scalar Int + + # The ` + "`" + `Float` + "`" + ` scalar type represents signed double-precision fractional values as specified by [IEEE 754](http://en.wikipedia.org/wiki/IEEE_floating_point). + scalar Float + + # The ` + "`" + `String` + "`" + ` scalar type represents textual data, represented as UTF-8 character sequences. The String type is most often used by GraphQL to represent free-form human-readable text. + scalar String + + # The ` + "`" + `Boolean` + "`" + ` scalar type represents ` + "`" + `true` + "`" + ` or ` + "`" + `false` + "`" + `. + scalar Boolean + + # The ` + "`" + `ID` + "`" + ` scalar type represents a unique identifier, often used to refetch an object or as key for a cache. The ID type appears in a JSON response as a String; however, it is not intended to be human-readable. When expected as an input type, any string (such as ` + "`" + `"4"` + "`" + `) or integer (such as ` + "`" + `4` + "`" + `) input value will be accepted as an ID. + scalar ID + + # Directs the executor to include this field or fragment only when the ` + "`" + `if` + "`" + ` argument is true. + directive @include( + # Included when true. + if: Boolean! + ) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT + + # Directs the executor to skip this field or fragment when the ` + "`" + `if` + "`" + ` argument is true. + directive @skip( + # Skipped when true. + if: Boolean! + ) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT + + # Marks an element of a GraphQL schema as no longer supported. + directive @deprecated( + # Explains why this element was deprecated, usually also including a suggestion + # for how to access supported similar data. Formatted in + # [Markdown](https://daringfireball.net/projects/markdown/). + reason: String = "No longer supported" + ) on FIELD_DEFINITION | ENUM_VALUE + + # A Directive provides a way to describe alternate runtime execution and type validation behavior in a GraphQL document. + # + # In some cases, you need to provide options to alter GraphQL's execution behavior + # in ways field arguments will not suffice, such as conditionally including or + # skipping a field. Directives provide this by describing additional information + # to the executor. + type __Directive { + name: String! + description: String + locations: [__DirectiveLocation!]! + args: [__InputValue!]! + } + + # A Directive can be adjacent to many parts of the GraphQL language, a + # __DirectiveLocation describes one such possible adjacencies. + enum __DirectiveLocation { + # Location adjacent to a query operation. + QUERY + # Location adjacent to a mutation operation. + MUTATION + # Location adjacent to a subscription operation. + SUBSCRIPTION + # Location adjacent to a field. + FIELD + # Location adjacent to a fragment definition. + FRAGMENT_DEFINITION + # Location adjacent to a fragment spread. + FRAGMENT_SPREAD + # Location adjacent to an inline fragment. + INLINE_FRAGMENT + # Location adjacent to a schema definition. + SCHEMA + # Location adjacent to a scalar definition. + SCALAR + # Location adjacent to an object type definition. + OBJECT + # Location adjacent to a field definition. + FIELD_DEFINITION + # Location adjacent to an argument definition. + ARGUMENT_DEFINITION + # Location adjacent to an interface definition. + INTERFACE + # Location adjacent to a union definition. + UNION + # Location adjacent to an enum definition. + ENUM + # Location adjacent to an enum value definition. + ENUM_VALUE + # Location adjacent to an input object type definition. + INPUT_OBJECT + # Location adjacent to an input object field definition. + INPUT_FIELD_DEFINITION + } + + # One possible value for a given Enum. Enum values are unique values, not a + # placeholder for a string or numeric value. However an Enum value is returned in + # a JSON response as a string. + type __EnumValue { + name: String! + description: String + isDeprecated: Boolean! + deprecationReason: String + } + + # Object and Interface types are described by a list of Fields, each of which has + # a name, potentially a list of arguments, and a return type. + type __Field { + name: String! + description: String + args: [__InputValue!]! + type: __Type! + isDeprecated: Boolean! + deprecationReason: String + } + + # Arguments provided to Fields or Directives and the input fields of an + # InputObject are represented as Input Values which describe their type and + # optionally a default value. + type __InputValue { + name: String! + description: String + type: __Type! + # A GraphQL-formatted string representing the default value for this input value. + defaultValue: String + } + + # A GraphQL Schema defines the capabilities of a GraphQL server. It exposes all + # available types and directives on the server, as well as the entry points for + # query, mutation, and subscription operations. + type __Schema { + # A list of all types supported by this server. + types: [__Type!]! + # The type that query operations will be rooted at. + queryType: __Type! + # If this server supports mutation, the type that mutation operations will be rooted at. + mutationType: __Type + # If this server support subscription, the type that subscription operations will be rooted at. + subscriptionType: __Type + # A list of all directives supported by this server. + directives: [__Directive!]! + } + + # The fundamental unit of any GraphQL Schema is the type. There are many kinds of + # types in GraphQL as represented by the ` + "`" + `__TypeKind` + "`" + ` enum. + # + # Depending on the kind of a type, certain fields describe information about that + # type. Scalar types provide no information beyond a name and description, while + # Enum types provide their values. Object and Interface types provide the fields + # they describe. Abstract types, Union and Interface, provide the Object types + # possible at runtime. List and NonNull types compose other types. + type __Type { + kind: __TypeKind! + name: String + description: String + fields(includeDeprecated: Boolean = false): [__Field!] + interfaces: [__Type!] + possibleTypes: [__Type!] + enumValues(includeDeprecated: Boolean = false): [__EnumValue!] + inputFields: [__InputValue!] + ofType: __Type + } + + # An enum describing what kind of type a given ` + "`" + `__Type` + "`" + ` is. + enum __TypeKind { + # Indicates this type is a scalar. + SCALAR + # Indicates this type is an object. ` + "`" + `fields` + "`" + ` and ` + "`" + `interfaces` + "`" + ` are valid fields. + OBJECT + # Indicates this type is an interface. ` + "`" + `fields` + "`" + ` and ` + "`" + `possibleTypes` + "`" + ` are valid fields. + INTERFACE + # Indicates this type is a union. ` + "`" + `possibleTypes` + "`" + ` is a valid field. + UNION + # Indicates this type is an enum. ` + "`" + `enumValues` + "`" + ` is a valid field. + ENUM + # Indicates this type is an input object. ` + "`" + `inputFields` + "`" + ` is a valid field. + INPUT_OBJECT + # Indicates this type is a list. ` + "`" + `ofType` + "`" + ` is a valid field. + LIST + # Indicates this type is a non-null. ` + "`" + `ofType` + "`" + ` is a valid field. + NON_NULL + } +` diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/schema/schema.go b/vendor/github.com/graph-gophers/graphql-go/internal/schema/schema.go new file mode 100644 index 0000000000..e549f17c07 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/schema/schema.go @@ -0,0 +1,570 @@ +package schema + +import ( + "fmt" + "text/scanner" + + "github.com/graph-gophers/graphql-go/errors" + "github.com/graph-gophers/graphql-go/internal/common" +) + +// Schema represents a GraphQL service's collective type system capabilities. +// A schema is defined in terms of the types and directives it supports as well as the root +// operation types for each kind of operation: `query`, `mutation`, and `subscription`. +// +// For a more formal definition, read the relevant section in the specification: +// +// http://facebook.github.io/graphql/draft/#sec-Schema +type Schema struct { + // EntryPoints determines the place in the type system where `query`, `mutation`, and + // `subscription` operations begin. + // + // http://facebook.github.io/graphql/draft/#sec-Root-Operation-Types + // + // NOTE: The specification refers to this concept as "Root Operation Types". + // TODO: Rename the `EntryPoints` field to `RootOperationTypes` to align with spec terminology. + EntryPoints map[string]NamedType + + // Types are the fundamental unit of any GraphQL schema. + // There are six kinds of named types, and two wrapping types. + // + // http://facebook.github.io/graphql/draft/#sec-Types + Types map[string]NamedType + + // TODO: Type extensions? + // http://facebook.github.io/graphql/draft/#sec-Type-Extensions + + // Directives are used to annotate various parts of a GraphQL document as an indicator that they + // should be evaluated differently by a validator, executor, or client tool such as a code + // generator. + // + // http://facebook.github.io/graphql/draft/#sec-Type-System.Directives + Directives map[string]*DirectiveDecl + + entryPointNames map[string]string + objects []*Object + unions []*Union + enums []*Enum +} + +// Resolve a named type in the schema by its name. +func (s *Schema) Resolve(name string) common.Type { + return s.Types[name] +} + +// NamedType represents a type with a name. +// +// http://facebook.github.io/graphql/draft/#NamedType +type NamedType interface { + common.Type + TypeName() string + Description() string +} + +// Scalar types represent primitive leaf values (e.g. a string or an integer) in a GraphQL type +// system. +// +// GraphQL responses take the form of a hierarchical tree; the leaves on these trees are GraphQL +// scalars. +// +// http://facebook.github.io/graphql/draft/#sec-Scalars +type Scalar struct { + Name string + Desc string + // TODO: Add a list of directives? +} + +// Object types represent a list of named fields, each of which yield a value of a specific type. +// +// GraphQL queries are hierarchical and composed, describing a tree of information. +// While Scalar types describe the leaf values of these hierarchical types, Objects describe the +// intermediate levels. +// +// http://facebook.github.io/graphql/draft/#sec-Objects +type Object struct { + Name string + Interfaces []*Interface + Fields FieldList + Desc string + // TODO: Add a list of directives? + + interfaceNames []string +} + +// Interface types represent a list of named fields and their arguments. +// +// GraphQL objects can then implement these interfaces which requires that the object type will +// define all fields defined by those interfaces. +// +// http://facebook.github.io/graphql/draft/#sec-Interfaces +type Interface struct { + Name string + PossibleTypes []*Object + Fields FieldList // NOTE: the spec refers to this as `FieldsDefinition`. + Desc string + // TODO: Add a list of directives? +} + +// Union types represent objects that could be one of a list of GraphQL object types, but provides no +// guaranteed fields between those types. +// +// They also differ from interfaces in that object types declare what interfaces they implement, but +// are not aware of what unions contain them. +// +// http://facebook.github.io/graphql/draft/#sec-Unions +type Union struct { + Name string + PossibleTypes []*Object // NOTE: the spec refers to this as `UnionMemberTypes`. + Desc string + // TODO: Add a list of directives? + + typeNames []string +} + +// Enum types describe a set of possible values. +// +// Like scalar types, Enum types also represent leaf values in a GraphQL type system. +// +// http://facebook.github.io/graphql/draft/#sec-Enums +type Enum struct { + Name string + Values []*EnumValue // NOTE: the spec refers to this as `EnumValuesDefinition`. + Desc string + // TODO: Add a list of directives? +} + +// EnumValue types are unique values that may be serialized as a string: the name of the +// represented value. +// +// http://facebook.github.io/graphql/draft/#EnumValueDefinition +type EnumValue struct { + Name string + Directives common.DirectiveList + Desc string + // TODO: Add a list of directives? +} + +// InputObject types define a set of input fields; the input fields are either scalars, enums, or +// other input objects. +// +// This allows arguments to accept arbitrarily complex structs. +// +// http://facebook.github.io/graphql/draft/#sec-Input-Objects +type InputObject struct { + Name string + Desc string + Values common.InputValueList + // TODO: Add a list of directives? +} + +// FieldsList is a list of an Object's Fields. +// +// http://facebook.github.io/graphql/draft/#FieldsDefinition +type FieldList []*Field + +// Get iterates over the field list, returning a pointer-to-Field when the field name matches the +// provided `name` argument. +// Returns nil when no field was found by that name. +func (l FieldList) Get(name string) *Field { + for _, f := range l { + if f.Name == name { + return f + } + } + return nil +} + +// Names returns a string slice of the field names in the FieldList. +func (l FieldList) Names() []string { + names := make([]string, len(l)) + for i, f := range l { + names[i] = f.Name + } + return names +} + +// http://facebook.github.io/graphql/draft/#sec-Type-System.Directives +type DirectiveDecl struct { + Name string + Desc string + Locs []string + Args common.InputValueList +} + +func (*Scalar) Kind() string { return "SCALAR" } +func (*Object) Kind() string { return "OBJECT" } +func (*Interface) Kind() string { return "INTERFACE" } +func (*Union) Kind() string { return "UNION" } +func (*Enum) Kind() string { return "ENUM" } +func (*InputObject) Kind() string { return "INPUT_OBJECT" } + +func (t *Scalar) String() string { return t.Name } +func (t *Object) String() string { return t.Name } +func (t *Interface) String() string { return t.Name } +func (t *Union) String() string { return t.Name } +func (t *Enum) String() string { return t.Name } +func (t *InputObject) String() string { return t.Name } + +func (t *Scalar) TypeName() string { return t.Name } +func (t *Object) TypeName() string { return t.Name } +func (t *Interface) TypeName() string { return t.Name } +func (t *Union) TypeName() string { return t.Name } +func (t *Enum) TypeName() string { return t.Name } +func (t *InputObject) TypeName() string { return t.Name } + +func (t *Scalar) Description() string { return t.Desc } +func (t *Object) Description() string { return t.Desc } +func (t *Interface) Description() string { return t.Desc } +func (t *Union) Description() string { return t.Desc } +func (t *Enum) Description() string { return t.Desc } +func (t *InputObject) Description() string { return t.Desc } + +// Field is a conceptual function which yields values. +// http://facebook.github.io/graphql/draft/#FieldDefinition +type Field struct { + Name string + Args common.InputValueList // NOTE: the spec refers to this as `ArgumentsDefinition`. + Type common.Type + Directives common.DirectiveList + Desc string +} + +// New initializes an instance of Schema. +func New() *Schema { + s := &Schema{ + entryPointNames: make(map[string]string), + Types: make(map[string]NamedType), + Directives: make(map[string]*DirectiveDecl), + } + for n, t := range Meta.Types { + s.Types[n] = t + } + for n, d := range Meta.Directives { + s.Directives[n] = d + } + return s +} + +// Parse the schema string. +func (s *Schema) Parse(schemaString string) error { + l := common.NewLexer(schemaString) + + err := l.CatchSyntaxError(func() { parseSchema(s, l) }) + if err != nil { + return err + } + + for _, t := range s.Types { + if err := resolveNamedType(s, t); err != nil { + return err + } + } + for _, d := range s.Directives { + for _, arg := range d.Args { + t, err := common.ResolveType(arg.Type, s.Resolve) + if err != nil { + return err + } + arg.Type = t + } + } + + s.EntryPoints = make(map[string]NamedType) + for key, name := range s.entryPointNames { + t, ok := s.Types[name] + if !ok { + if !ok { + return errors.Errorf("type %q not found", name) + } + } + s.EntryPoints[key] = t + } + + for _, obj := range s.objects { + obj.Interfaces = make([]*Interface, len(obj.interfaceNames)) + for i, intfName := range obj.interfaceNames { + t, ok := s.Types[intfName] + if !ok { + return errors.Errorf("interface %q not found", intfName) + } + intf, ok := t.(*Interface) + if !ok { + return errors.Errorf("type %q is not an interface", intfName) + } + obj.Interfaces[i] = intf + intf.PossibleTypes = append(intf.PossibleTypes, obj) + } + } + + for _, union := range s.unions { + union.PossibleTypes = make([]*Object, len(union.typeNames)) + for i, name := range union.typeNames { + t, ok := s.Types[name] + if !ok { + return errors.Errorf("object type %q not found", name) + } + obj, ok := t.(*Object) + if !ok { + return errors.Errorf("type %q is not an object", name) + } + union.PossibleTypes[i] = obj + } + } + + for _, enum := range s.enums { + for _, value := range enum.Values { + if err := resolveDirectives(s, value.Directives); err != nil { + return err + } + } + } + + return nil +} + +func resolveNamedType(s *Schema, t NamedType) error { + switch t := t.(type) { + case *Object: + for _, f := range t.Fields { + if err := resolveField(s, f); err != nil { + return err + } + } + case *Interface: + for _, f := range t.Fields { + if err := resolveField(s, f); err != nil { + return err + } + } + case *InputObject: + if err := resolveInputObject(s, t.Values); err != nil { + return err + } + } + return nil +} + +func resolveField(s *Schema, f *Field) error { + t, err := common.ResolveType(f.Type, s.Resolve) + if err != nil { + return err + } + f.Type = t + if err := resolveDirectives(s, f.Directives); err != nil { + return err + } + return resolveInputObject(s, f.Args) +} + +func resolveDirectives(s *Schema, directives common.DirectiveList) error { + for _, d := range directives { + dirName := d.Name.Name + dd, ok := s.Directives[dirName] + if !ok { + return errors.Errorf("directive %q not found", dirName) + } + for _, arg := range d.Args { + if dd.Args.Get(arg.Name.Name) == nil { + return errors.Errorf("invalid argument %q for directive %q", arg.Name.Name, dirName) + } + } + for _, arg := range dd.Args { + if _, ok := d.Args.Get(arg.Name.Name); !ok { + d.Args = append(d.Args, common.Argument{Name: arg.Name, Value: arg.Default}) + } + } + } + return nil +} + +func resolveInputObject(s *Schema, values common.InputValueList) error { + for _, v := range values { + t, err := common.ResolveType(v.Type, s.Resolve) + if err != nil { + return err + } + v.Type = t + } + return nil +} + +func parseSchema(s *Schema, l *common.Lexer) { + l.Consume() + + for l.Peek() != scanner.EOF { + desc := l.DescComment() + switch x := l.ConsumeIdent(); x { + + case "schema": + l.ConsumeToken('{') + for l.Peek() != '}' { + name := l.ConsumeIdent() + l.ConsumeToken(':') + typ := l.ConsumeIdent() + s.entryPointNames[name] = typ + } + l.ConsumeToken('}') + + case "type": + obj := parseObjectDef(l) + obj.Desc = desc + s.Types[obj.Name] = obj + s.objects = append(s.objects, obj) + + case "interface": + iface := parseInterfaceDef(l) + iface.Desc = desc + s.Types[iface.Name] = iface + + case "union": + union := parseUnionDef(l) + union.Desc = desc + s.Types[union.Name] = union + s.unions = append(s.unions, union) + + case "enum": + enum := parseEnumDef(l) + enum.Desc = desc + s.Types[enum.Name] = enum + s.enums = append(s.enums, enum) + + case "input": + input := parseInputDef(l) + input.Desc = desc + s.Types[input.Name] = input + + case "scalar": + name := l.ConsumeIdent() + s.Types[name] = &Scalar{Name: name, Desc: desc} + + case "directive": + directive := parseDirectiveDef(l) + directive.Desc = desc + s.Directives[directive.Name] = directive + + default: + // TODO: Add support for type extensions. + l.SyntaxError(fmt.Sprintf(`unexpected %q, expecting "schema", "type", "enum", "interface", "union", "input", "scalar" or "directive"`, x)) + } + } +} + +func parseObjectDef(l *common.Lexer) *Object { + object := &Object{Name: l.ConsumeIdent()} + + if l.Peek() == scanner.Ident { + l.ConsumeKeyword("implements") + + for l.Peek() != '{' { + if l.Peek() == '&' { + l.ConsumeToken('&') + } + + object.interfaceNames = append(object.interfaceNames, l.ConsumeIdent()) + } + } + + l.ConsumeToken('{') + object.Fields = parseFieldsDef(l) + l.ConsumeToken('}') + + return object +} + +func parseInterfaceDef(l *common.Lexer) *Interface { + i := &Interface{Name: l.ConsumeIdent()} + + l.ConsumeToken('{') + i.Fields = parseFieldsDef(l) + l.ConsumeToken('}') + + return i +} + +func parseUnionDef(l *common.Lexer) *Union { + union := &Union{Name: l.ConsumeIdent()} + + l.ConsumeToken('=') + union.typeNames = []string{l.ConsumeIdent()} + for l.Peek() == '|' { + l.ConsumeToken('|') + union.typeNames = append(union.typeNames, l.ConsumeIdent()) + } + + return union +} + +func parseInputDef(l *common.Lexer) *InputObject { + i := &InputObject{} + i.Name = l.ConsumeIdent() + l.ConsumeToken('{') + for l.Peek() != '}' { + i.Values = append(i.Values, common.ParseInputValue(l)) + } + l.ConsumeToken('}') + return i +} + +func parseEnumDef(l *common.Lexer) *Enum { + enum := &Enum{Name: l.ConsumeIdent()} + + l.ConsumeToken('{') + for l.Peek() != '}' { + v := &EnumValue{ + Desc: l.DescComment(), + Name: l.ConsumeIdent(), + Directives: common.ParseDirectives(l), + } + + enum.Values = append(enum.Values, v) + } + l.ConsumeToken('}') + return enum +} + +func parseDirectiveDef(l *common.Lexer) *DirectiveDecl { + l.ConsumeToken('@') + d := &DirectiveDecl{Name: l.ConsumeIdent()} + + if l.Peek() == '(' { + l.ConsumeToken('(') + for l.Peek() != ')' { + v := common.ParseInputValue(l) + d.Args = append(d.Args, v) + } + l.ConsumeToken(')') + } + + l.ConsumeKeyword("on") + + for { + loc := l.ConsumeIdent() + d.Locs = append(d.Locs, loc) + if l.Peek() != '|' { + break + } + l.ConsumeToken('|') + } + return d +} + +func parseFieldsDef(l *common.Lexer) FieldList { + var fields FieldList + for l.Peek() != '}' { + f := &Field{} + f.Desc = l.DescComment() + f.Name = l.ConsumeIdent() + if l.Peek() == '(' { + l.ConsumeToken('(') + for l.Peek() != ')' { + f.Args = append(f.Args, common.ParseInputValue(l)) + } + l.ConsumeToken(')') + } + l.ConsumeToken(':') + f.Type = common.ParseType(l) + f.Directives = common.ParseDirectives(l) + fields = append(fields, f) + } + return fields +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/validation/suggestion.go b/vendor/github.com/graph-gophers/graphql-go/internal/validation/suggestion.go new file mode 100644 index 0000000000..9702b5f529 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/validation/suggestion.go @@ -0,0 +1,71 @@ +package validation + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +func makeSuggestion(prefix string, options []string, input string) string { + var selected []string + distances := make(map[string]int) + for _, opt := range options { + distance := levenshteinDistance(input, opt) + threshold := max(len(input)/2, max(len(opt)/2, 1)) + if distance < threshold { + selected = append(selected, opt) + distances[opt] = distance + } + } + + if len(selected) == 0 { + return "" + } + sort.Slice(selected, func(i, j int) bool { + return distances[selected[i]] < distances[selected[j]] + }) + + parts := make([]string, len(selected)) + for i, opt := range selected { + parts[i] = strconv.Quote(opt) + } + if len(parts) > 1 { + parts[len(parts)-1] = "or " + parts[len(parts)-1] + } + return fmt.Sprintf(" %s %s?", prefix, strings.Join(parts, ", ")) +} + +func levenshteinDistance(s1, s2 string) int { + column := make([]int, len(s1)+1) + for y := range s1 { + column[y+1] = y + 1 + } + for x, rx := range s2 { + column[0] = x + 1 + lastdiag := x + for y, ry := range s1 { + olddiag := column[y+1] + if rx != ry { + lastdiag++ + } + column[y+1] = min(column[y+1]+1, min(column[y]+1, lastdiag)) + lastdiag = olddiag + } + } + return column[len(s1)] +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} diff --git a/vendor/github.com/graph-gophers/graphql-go/internal/validation/validation.go b/vendor/github.com/graph-gophers/graphql-go/internal/validation/validation.go new file mode 100644 index 0000000000..94ad5ca7fb --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/internal/validation/validation.go @@ -0,0 +1,909 @@ +package validation + +import ( + "fmt" + "math" + "reflect" + "strconv" + "strings" + "text/scanner" + + "github.com/graph-gophers/graphql-go/errors" + "github.com/graph-gophers/graphql-go/internal/common" + "github.com/graph-gophers/graphql-go/internal/query" + "github.com/graph-gophers/graphql-go/internal/schema" +) + +type varSet map[*common.InputValue]struct{} + +type selectionPair struct{ a, b query.Selection } + +type fieldInfo struct { + sf *schema.Field + parent schema.NamedType +} + +type context struct { + schema *schema.Schema + doc *query.Document + errs []*errors.QueryError + opErrs map[*query.Operation][]*errors.QueryError + usedVars map[*query.Operation]varSet + fieldMap map[*query.Field]fieldInfo + overlapValidated map[selectionPair]struct{} + maxDepth int +} + +func (c *context) addErr(loc errors.Location, rule string, format string, a ...interface{}) { + c.addErrMultiLoc([]errors.Location{loc}, rule, format, a...) +} + +func (c *context) addErrMultiLoc(locs []errors.Location, rule string, format string, a ...interface{}) { + c.errs = append(c.errs, &errors.QueryError{ + Message: fmt.Sprintf(format, a...), + Locations: locs, + Rule: rule, + }) +} + +type opContext struct { + *context + ops []*query.Operation +} + +func newContext(s *schema.Schema, doc *query.Document, maxDepth int) *context { + return &context{ + schema: s, + doc: doc, + opErrs: make(map[*query.Operation][]*errors.QueryError), + usedVars: make(map[*query.Operation]varSet), + fieldMap: make(map[*query.Field]fieldInfo), + overlapValidated: make(map[selectionPair]struct{}), + maxDepth: maxDepth, + } +} + +func Validate(s *schema.Schema, doc *query.Document, maxDepth int) []*errors.QueryError { + c := newContext(s, doc, maxDepth) + + opNames := make(nameSet) + fragUsedBy := make(map[*query.FragmentDecl][]*query.Operation) + for _, op := range doc.Operations { + c.usedVars[op] = make(varSet) + opc := &opContext{c, []*query.Operation{op}} + + // Check if max depth is exceeded, if it's set. If max depth is exceeded, + // don't continue to validate the document and exit early. + if validateMaxDepth(opc, op.Selections, 1) { + return c.errs + } + + if op.Name.Name == "" && len(doc.Operations) != 1 { + c.addErr(op.Loc, "LoneAnonymousOperation", "This anonymous operation must be the only defined operation.") + } + if op.Name.Name != "" { + validateName(c, opNames, op.Name, "UniqueOperationNames", "operation") + } + + validateDirectives(opc, string(op.Type), op.Directives) + + varNames := make(nameSet) + for _, v := range op.Vars { + validateName(c, varNames, v.Name, "UniqueVariableNames", "variable") + + t := resolveType(c, v.Type) + if !canBeInput(t) { + c.addErr(v.TypeLoc, "VariablesAreInputTypes", "Variable %q cannot be non-input type %q.", "$"+v.Name.Name, t) + } + + if v.Default != nil { + validateLiteral(opc, v.Default) + + if t != nil { + if nn, ok := t.(*common.NonNull); ok { + c.addErr(v.Default.Location(), "DefaultValuesOfCorrectType", "Variable %q of type %q is required and will not use the default value. Perhaps you meant to use type %q.", "$"+v.Name.Name, t, nn.OfType) + } + + if ok, reason := validateValueType(opc, v.Default, t); !ok { + c.addErr(v.Default.Location(), "DefaultValuesOfCorrectType", "Variable %q of type %q has invalid default value %s.\n%s", "$"+v.Name.Name, t, v.Default, reason) + } + } + } + } + + var entryPoint schema.NamedType + switch op.Type { + case query.Query: + entryPoint = s.EntryPoints["query"] + case query.Mutation: + entryPoint = s.EntryPoints["mutation"] + case query.Subscription: + entryPoint = s.EntryPoints["subscription"] + default: + panic("unreachable") + } + + validateSelectionSet(opc, op.Selections, entryPoint) + + fragUsed := make(map[*query.FragmentDecl]struct{}) + markUsedFragments(c, op.Selections, fragUsed) + for frag := range fragUsed { + fragUsedBy[frag] = append(fragUsedBy[frag], op) + } + } + + fragNames := make(nameSet) + fragVisited := make(map[*query.FragmentDecl]struct{}) + for _, frag := range doc.Fragments { + opc := &opContext{c, fragUsedBy[frag]} + + validateName(c, fragNames, frag.Name, "UniqueFragmentNames", "fragment") + validateDirectives(opc, "FRAGMENT_DEFINITION", frag.Directives) + + t := unwrapType(resolveType(c, &frag.On)) + // continue even if t is nil + if t != nil && !canBeFragment(t) { + c.addErr(frag.On.Loc, "FragmentsOnCompositeTypes", "Fragment %q cannot condition on non composite type %q.", frag.Name.Name, t) + continue + } + + validateSelectionSet(opc, frag.Selections, t) + + if _, ok := fragVisited[frag]; !ok { + detectFragmentCycle(c, frag.Selections, fragVisited, nil, map[string]int{frag.Name.Name: 0}) + } + } + + for _, frag := range doc.Fragments { + if len(fragUsedBy[frag]) == 0 { + c.addErr(frag.Loc, "NoUnusedFragments", "Fragment %q is never used.", frag.Name.Name) + } + } + + for _, op := range doc.Operations { + c.errs = append(c.errs, c.opErrs[op]...) + + opUsedVars := c.usedVars[op] + for _, v := range op.Vars { + if _, ok := opUsedVars[v]; !ok { + opSuffix := "" + if op.Name.Name != "" { + opSuffix = fmt.Sprintf(" in operation %q", op.Name.Name) + } + c.addErr(v.Loc, "NoUnusedVariables", "Variable %q is never used%s.", "$"+v.Name.Name, opSuffix) + } + } + } + + return c.errs +} + +// validates the query doesn't go deeper than maxDepth (if set). Returns whether +// or not query validated max depth to avoid excessive recursion. +func validateMaxDepth(c *opContext, sels []query.Selection, depth int) bool { + // maxDepth checking is turned off when maxDepth is 0 + if c.maxDepth == 0 { + return false + } + + exceededMaxDepth := false + + for _, sel := range sels { + switch sel := sel.(type) { + case *query.Field: + if depth > c.maxDepth { + exceededMaxDepth = true + c.addErr(sel.Alias.Loc, "MaxDepthExceeded", "Field %q has depth %d that exceeds max depth %d", sel.Name.Name, depth, c.maxDepth) + continue + } + exceededMaxDepth = exceededMaxDepth || validateMaxDepth(c, sel.Selections, depth+1) + case *query.InlineFragment: + // Depth is not checked because inline fragments resolve to other fields which are checked. + // Depth is not incremented because inline fragments have the same depth as neighboring fields + exceededMaxDepth = exceededMaxDepth || validateMaxDepth(c, sel.Selections, depth) + case *query.FragmentSpread: + // Depth is not checked because fragments resolve to other fields which are checked. + frag := c.doc.Fragments.Get(sel.Name.Name) + if frag == nil { + // In case of unknown fragment (invalid request), ignore max depth evaluation + c.addErr(sel.Loc, "MaxDepthEvaluationError", "Unknown fragment %q. Unable to evaluate depth.", sel.Name.Name) + continue + } + // Depth is not incremented because fragments have the same depth as surrounding fields + exceededMaxDepth = exceededMaxDepth || validateMaxDepth(c, frag.Selections, depth) + } + } + + return exceededMaxDepth +} + +func validateSelectionSet(c *opContext, sels []query.Selection, t schema.NamedType) { + for _, sel := range sels { + validateSelection(c, sel, t) + } + + for i, a := range sels { + for _, b := range sels[i+1:] { + c.validateOverlap(a, b, nil, nil) + } + } +} + +func validateSelection(c *opContext, sel query.Selection, t schema.NamedType) { + switch sel := sel.(type) { + case *query.Field: + validateDirectives(c, "FIELD", sel.Directives) + + fieldName := sel.Name.Name + var f *schema.Field + switch fieldName { + case "__typename": + f = &schema.Field{ + Name: "__typename", + Type: c.schema.Types["String"], + } + case "__schema": + f = &schema.Field{ + Name: "__schema", + Type: c.schema.Types["__Schema"], + } + case "__type": + f = &schema.Field{ + Name: "__type", + Args: common.InputValueList{ + &common.InputValue{ + Name: common.Ident{Name: "name"}, + Type: &common.NonNull{OfType: c.schema.Types["String"]}, + }, + }, + Type: c.schema.Types["__Type"], + } + default: + f = fields(t).Get(fieldName) + if f == nil && t != nil { + suggestion := makeSuggestion("Did you mean", fields(t).Names(), fieldName) + c.addErr(sel.Alias.Loc, "FieldsOnCorrectType", "Cannot query field %q on type %q.%s", fieldName, t, suggestion) + } + } + c.fieldMap[sel] = fieldInfo{sf: f, parent: t} + + validateArgumentLiterals(c, sel.Arguments) + if f != nil { + validateArgumentTypes(c, sel.Arguments, f.Args, sel.Alias.Loc, + func() string { return fmt.Sprintf("field %q of type %q", fieldName, t) }, + func() string { return fmt.Sprintf("Field %q", fieldName) }, + ) + } + + var ft common.Type + if f != nil { + ft = f.Type + sf := hasSubfields(ft) + if sf && sel.Selections == nil { + c.addErr(sel.Alias.Loc, "ScalarLeafs", "Field %q of type %q must have a selection of subfields. Did you mean \"%s { ... }\"?", fieldName, ft, fieldName) + } + if !sf && sel.Selections != nil { + c.addErr(sel.SelectionSetLoc, "ScalarLeafs", "Field %q must not have a selection since type %q has no subfields.", fieldName, ft) + } + } + if sel.Selections != nil { + validateSelectionSet(c, sel.Selections, unwrapType(ft)) + } + + case *query.InlineFragment: + validateDirectives(c, "INLINE_FRAGMENT", sel.Directives) + if sel.On.Name != "" { + fragTyp := unwrapType(resolveType(c.context, &sel.On)) + if fragTyp != nil && !compatible(t, fragTyp) { + c.addErr(sel.Loc, "PossibleFragmentSpreads", "Fragment cannot be spread here as objects of type %q can never be of type %q.", t, fragTyp) + } + t = fragTyp + // continue even if t is nil + } + if t != nil && !canBeFragment(t) { + c.addErr(sel.On.Loc, "FragmentsOnCompositeTypes", "Fragment cannot condition on non composite type %q.", t) + return + } + validateSelectionSet(c, sel.Selections, unwrapType(t)) + + case *query.FragmentSpread: + validateDirectives(c, "FRAGMENT_SPREAD", sel.Directives) + frag := c.doc.Fragments.Get(sel.Name.Name) + if frag == nil { + c.addErr(sel.Name.Loc, "KnownFragmentNames", "Unknown fragment %q.", sel.Name.Name) + return + } + fragTyp := c.schema.Types[frag.On.Name] + if !compatible(t, fragTyp) { + c.addErr(sel.Loc, "PossibleFragmentSpreads", "Fragment %q cannot be spread here as objects of type %q can never be of type %q.", frag.Name.Name, t, fragTyp) + } + + default: + panic("unreachable") + } +} + +func compatible(a, b common.Type) bool { + for _, pta := range possibleTypes(a) { + for _, ptb := range possibleTypes(b) { + if pta == ptb { + return true + } + } + } + return false +} + +func possibleTypes(t common.Type) []*schema.Object { + switch t := t.(type) { + case *schema.Object: + return []*schema.Object{t} + case *schema.Interface: + return t.PossibleTypes + case *schema.Union: + return t.PossibleTypes + default: + return nil + } +} + +func markUsedFragments(c *context, sels []query.Selection, fragUsed map[*query.FragmentDecl]struct{}) { + for _, sel := range sels { + switch sel := sel.(type) { + case *query.Field: + if sel.Selections != nil { + markUsedFragments(c, sel.Selections, fragUsed) + } + + case *query.InlineFragment: + markUsedFragments(c, sel.Selections, fragUsed) + + case *query.FragmentSpread: + frag := c.doc.Fragments.Get(sel.Name.Name) + if frag == nil { + return + } + + if _, ok := fragUsed[frag]; ok { + return + } + fragUsed[frag] = struct{}{} + markUsedFragments(c, frag.Selections, fragUsed) + + default: + panic("unreachable") + } + } +} + +func detectFragmentCycle(c *context, sels []query.Selection, fragVisited map[*query.FragmentDecl]struct{}, spreadPath []*query.FragmentSpread, spreadPathIndex map[string]int) { + for _, sel := range sels { + detectFragmentCycleSel(c, sel, fragVisited, spreadPath, spreadPathIndex) + } +} + +func detectFragmentCycleSel(c *context, sel query.Selection, fragVisited map[*query.FragmentDecl]struct{}, spreadPath []*query.FragmentSpread, spreadPathIndex map[string]int) { + switch sel := sel.(type) { + case *query.Field: + if sel.Selections != nil { + detectFragmentCycle(c, sel.Selections, fragVisited, spreadPath, spreadPathIndex) + } + + case *query.InlineFragment: + detectFragmentCycle(c, sel.Selections, fragVisited, spreadPath, spreadPathIndex) + + case *query.FragmentSpread: + frag := c.doc.Fragments.Get(sel.Name.Name) + if frag == nil { + return + } + + spreadPath = append(spreadPath, sel) + if i, ok := spreadPathIndex[frag.Name.Name]; ok { + cyclePath := spreadPath[i:] + via := "" + if len(cyclePath) > 1 { + names := make([]string, len(cyclePath)-1) + for i, frag := range cyclePath[:len(cyclePath)-1] { + names[i] = frag.Name.Name + } + via = " via " + strings.Join(names, ", ") + } + + locs := make([]errors.Location, len(cyclePath)) + for i, frag := range cyclePath { + locs[i] = frag.Loc + } + c.addErrMultiLoc(locs, "NoFragmentCycles", "Cannot spread fragment %q within itself%s.", frag.Name.Name, via) + return + } + + if _, ok := fragVisited[frag]; ok { + return + } + fragVisited[frag] = struct{}{} + + spreadPathIndex[frag.Name.Name] = len(spreadPath) + detectFragmentCycle(c, frag.Selections, fragVisited, spreadPath, spreadPathIndex) + delete(spreadPathIndex, frag.Name.Name) + + default: + panic("unreachable") + } +} + +func (c *context) validateOverlap(a, b query.Selection, reasons *[]string, locs *[]errors.Location) { + if a == b { + return + } + + if _, ok := c.overlapValidated[selectionPair{a, b}]; ok { + return + } + c.overlapValidated[selectionPair{a, b}] = struct{}{} + c.overlapValidated[selectionPair{b, a}] = struct{}{} + + switch a := a.(type) { + case *query.Field: + switch b := b.(type) { + case *query.Field: + if b.Alias.Loc.Before(a.Alias.Loc) { + a, b = b, a + } + if reasons2, locs2 := c.validateFieldOverlap(a, b); len(reasons2) != 0 { + locs2 = append(locs2, a.Alias.Loc, b.Alias.Loc) + if reasons == nil { + c.addErrMultiLoc(locs2, "OverlappingFieldsCanBeMerged", "Fields %q conflict because %s. Use different aliases on the fields to fetch both if this was intentional.", a.Alias.Name, strings.Join(reasons2, " and ")) + return + } + for _, r := range reasons2 { + *reasons = append(*reasons, fmt.Sprintf("subfields %q conflict because %s", a.Alias.Name, r)) + } + *locs = append(*locs, locs2...) + } + + case *query.InlineFragment: + for _, sel := range b.Selections { + c.validateOverlap(a, sel, reasons, locs) + } + + case *query.FragmentSpread: + if frag := c.doc.Fragments.Get(b.Name.Name); frag != nil { + for _, sel := range frag.Selections { + c.validateOverlap(a, sel, reasons, locs) + } + } + + default: + panic("unreachable") + } + + case *query.InlineFragment: + for _, sel := range a.Selections { + c.validateOverlap(sel, b, reasons, locs) + } + + case *query.FragmentSpread: + if frag := c.doc.Fragments.Get(a.Name.Name); frag != nil { + for _, sel := range frag.Selections { + c.validateOverlap(sel, b, reasons, locs) + } + } + + default: + panic("unreachable") + } +} + +func (c *context) validateFieldOverlap(a, b *query.Field) ([]string, []errors.Location) { + if a.Alias.Name != b.Alias.Name { + return nil, nil + } + + if asf := c.fieldMap[a].sf; asf != nil { + if bsf := c.fieldMap[b].sf; bsf != nil { + if !typesCompatible(asf.Type, bsf.Type) { + return []string{fmt.Sprintf("they return conflicting types %s and %s", asf.Type, bsf.Type)}, nil + } + } + } + + at := c.fieldMap[a].parent + bt := c.fieldMap[b].parent + if at == nil || bt == nil || at == bt { + if a.Name.Name != b.Name.Name { + return []string{fmt.Sprintf("%s and %s are different fields", a.Name.Name, b.Name.Name)}, nil + } + + if argumentsConflict(a.Arguments, b.Arguments) { + return []string{"they have differing arguments"}, nil + } + } + + var reasons []string + var locs []errors.Location + for _, a2 := range a.Selections { + for _, b2 := range b.Selections { + c.validateOverlap(a2, b2, &reasons, &locs) + } + } + return reasons, locs +} + +func argumentsConflict(a, b common.ArgumentList) bool { + if len(a) != len(b) { + return true + } + for _, argA := range a { + valB, ok := b.Get(argA.Name.Name) + if !ok || !reflect.DeepEqual(argA.Value.Value(nil), valB.Value(nil)) { + return true + } + } + return false +} + +func fields(t common.Type) schema.FieldList { + switch t := t.(type) { + case *schema.Object: + return t.Fields + case *schema.Interface: + return t.Fields + default: + return nil + } +} + +func unwrapType(t common.Type) schema.NamedType { + if t == nil { + return nil + } + for { + switch t2 := t.(type) { + case schema.NamedType: + return t2 + case *common.List: + t = t2.OfType + case *common.NonNull: + t = t2.OfType + default: + panic("unreachable") + } + } +} + +func resolveType(c *context, t common.Type) common.Type { + t2, err := common.ResolveType(t, c.schema.Resolve) + if err != nil { + c.errs = append(c.errs, err) + } + return t2 +} + +func validateDirectives(c *opContext, loc string, directives common.DirectiveList) { + directiveNames := make(nameSet) + for _, d := range directives { + dirName := d.Name.Name + validateNameCustomMsg(c.context, directiveNames, d.Name, "UniqueDirectivesPerLocation", func() string { + return fmt.Sprintf("The directive %q can only be used once at this location.", dirName) + }) + + validateArgumentLiterals(c, d.Args) + + dd, ok := c.schema.Directives[dirName] + if !ok { + c.addErr(d.Name.Loc, "KnownDirectives", "Unknown directive %q.", dirName) + continue + } + + locOK := false + for _, allowedLoc := range dd.Locs { + if loc == allowedLoc { + locOK = true + break + } + } + if !locOK { + c.addErr(d.Name.Loc, "KnownDirectives", "Directive %q may not be used on %s.", dirName, loc) + } + + validateArgumentTypes(c, d.Args, dd.Args, d.Name.Loc, + func() string { return fmt.Sprintf("directive %q", "@"+dirName) }, + func() string { return fmt.Sprintf("Directive %q", "@"+dirName) }, + ) + } +} + +type nameSet map[string]errors.Location + +func validateName(c *context, set nameSet, name common.Ident, rule string, kind string) { + validateNameCustomMsg(c, set, name, rule, func() string { + return fmt.Sprintf("There can be only one %s named %q.", kind, name.Name) + }) +} + +func validateNameCustomMsg(c *context, set nameSet, name common.Ident, rule string, msg func() string) { + if loc, ok := set[name.Name]; ok { + c.addErrMultiLoc([]errors.Location{loc, name.Loc}, rule, msg()) + return + } + set[name.Name] = name.Loc +} + +func validateArgumentTypes(c *opContext, args common.ArgumentList, argDecls common.InputValueList, loc errors.Location, owner1, owner2 func() string) { + for _, selArg := range args { + arg := argDecls.Get(selArg.Name.Name) + if arg == nil { + c.addErr(selArg.Name.Loc, "KnownArgumentNames", "Unknown argument %q on %s.", selArg.Name.Name, owner1()) + continue + } + value := selArg.Value + if ok, reason := validateValueType(c, value, arg.Type); !ok { + c.addErr(value.Location(), "ArgumentsOfCorrectType", "Argument %q has invalid value %s.\n%s", arg.Name.Name, value, reason) + } + } + for _, decl := range argDecls { + if _, ok := decl.Type.(*common.NonNull); ok { + if _, ok := args.Get(decl.Name.Name); !ok { + c.addErr(loc, "ProvidedNonNullArguments", "%s argument %q of type %q is required but not provided.", owner2(), decl.Name.Name, decl.Type) + } + } + } +} + +func validateArgumentLiterals(c *opContext, args common.ArgumentList) { + argNames := make(nameSet) + for _, arg := range args { + validateName(c.context, argNames, arg.Name, "UniqueArgumentNames", "argument") + validateLiteral(c, arg.Value) + } +} + +func validateLiteral(c *opContext, l common.Literal) { + switch l := l.(type) { + case *common.ObjectLit: + fieldNames := make(nameSet) + for _, f := range l.Fields { + validateName(c.context, fieldNames, f.Name, "UniqueInputFieldNames", "input field") + validateLiteral(c, f.Value) + } + case *common.ListLit: + for _, entry := range l.Entries { + validateLiteral(c, entry) + } + case *common.Variable: + for _, op := range c.ops { + v := op.Vars.Get(l.Name) + if v == nil { + byOp := "" + if op.Name.Name != "" { + byOp = fmt.Sprintf(" by operation %q", op.Name.Name) + } + c.opErrs[op] = append(c.opErrs[op], &errors.QueryError{ + Message: fmt.Sprintf("Variable %q is not defined%s.", "$"+l.Name, byOp), + Locations: []errors.Location{l.Loc, op.Loc}, + Rule: "NoUndefinedVariables", + }) + continue + } + c.usedVars[op][v] = struct{}{} + } + } +} + +func validateValueType(c *opContext, v common.Literal, t common.Type) (bool, string) { + if v, ok := v.(*common.Variable); ok { + for _, op := range c.ops { + if v2 := op.Vars.Get(v.Name); v2 != nil { + t2, err := common.ResolveType(v2.Type, c.schema.Resolve) + if _, ok := t2.(*common.NonNull); !ok && v2.Default != nil { + t2 = &common.NonNull{OfType: t2} + } + if err == nil && !typeCanBeUsedAs(t2, t) { + c.addErrMultiLoc([]errors.Location{v2.Loc, v.Loc}, "VariablesInAllowedPosition", "Variable %q of type %q used in position expecting type %q.", "$"+v.Name, t2, t) + } + } + } + return true, "" + } + + if nn, ok := t.(*common.NonNull); ok { + if isNull(v) { + return false, fmt.Sprintf("Expected %q, found null.", t) + } + t = nn.OfType + } + if isNull(v) { + return true, "" + } + + switch t := t.(type) { + case *schema.Scalar, *schema.Enum: + if lit, ok := v.(*common.BasicLit); ok { + if validateBasicLit(lit, t) { + return true, "" + } + } + + case *common.List: + list, ok := v.(*common.ListLit) + if !ok { + return validateValueType(c, v, t.OfType) // single value instead of list + } + for i, entry := range list.Entries { + if ok, reason := validateValueType(c, entry, t.OfType); !ok { + return false, fmt.Sprintf("In element #%d: %s", i, reason) + } + } + return true, "" + + case *schema.InputObject: + v, ok := v.(*common.ObjectLit) + if !ok { + return false, fmt.Sprintf("Expected %q, found not an object.", t) + } + for _, f := range v.Fields { + name := f.Name.Name + iv := t.Values.Get(name) + if iv == nil { + return false, fmt.Sprintf("In field %q: Unknown field.", name) + } + if ok, reason := validateValueType(c, f.Value, iv.Type); !ok { + return false, fmt.Sprintf("In field %q: %s", name, reason) + } + } + for _, iv := range t.Values { + found := false + for _, f := range v.Fields { + if f.Name.Name == iv.Name.Name { + found = true + break + } + } + if !found { + if _, ok := iv.Type.(*common.NonNull); ok && iv.Default == nil { + return false, fmt.Sprintf("In field %q: Expected %q, found null.", iv.Name.Name, iv.Type) + } + } + } + return true, "" + } + + return false, fmt.Sprintf("Expected type %q, found %s.", t, v) +} + +func validateBasicLit(v *common.BasicLit, t common.Type) bool { + switch t := t.(type) { + case *schema.Scalar: + switch t.Name { + case "Int": + if v.Type != scanner.Int { + return false + } + f, err := strconv.ParseFloat(v.Text, 64) + if err != nil { + panic(err) + } + return f >= math.MinInt32 && f <= math.MaxInt32 + case "Float": + return v.Type == scanner.Int || v.Type == scanner.Float + case "String": + return v.Type == scanner.String + case "Boolean": + return v.Type == scanner.Ident && (v.Text == "true" || v.Text == "false") + case "ID": + return v.Type == scanner.Int || v.Type == scanner.String + default: + //TODO: Type-check against expected type by Unmarshalling + return true + } + + case *schema.Enum: + if v.Type != scanner.Ident { + return false + } + for _, option := range t.Values { + if option.Name == v.Text { + return true + } + } + return false + } + + return false +} + +func canBeFragment(t common.Type) bool { + switch t.(type) { + case *schema.Object, *schema.Interface, *schema.Union: + return true + default: + return false + } +} + +func canBeInput(t common.Type) bool { + switch t := t.(type) { + case *schema.InputObject, *schema.Scalar, *schema.Enum: + return true + case *common.List: + return canBeInput(t.OfType) + case *common.NonNull: + return canBeInput(t.OfType) + default: + return false + } +} + +func hasSubfields(t common.Type) bool { + switch t := t.(type) { + case *schema.Object, *schema.Interface, *schema.Union: + return true + case *common.List: + return hasSubfields(t.OfType) + case *common.NonNull: + return hasSubfields(t.OfType) + default: + return false + } +} + +func isLeaf(t common.Type) bool { + switch t.(type) { + case *schema.Scalar, *schema.Enum: + return true + default: + return false + } +} + +func isNull(lit interface{}) bool { + _, ok := lit.(*common.NullLit) + return ok +} + +func typesCompatible(a, b common.Type) bool { + al, aIsList := a.(*common.List) + bl, bIsList := b.(*common.List) + if aIsList || bIsList { + return aIsList && bIsList && typesCompatible(al.OfType, bl.OfType) + } + + ann, aIsNN := a.(*common.NonNull) + bnn, bIsNN := b.(*common.NonNull) + if aIsNN || bIsNN { + return aIsNN && bIsNN && typesCompatible(ann.OfType, bnn.OfType) + } + + if isLeaf(a) || isLeaf(b) { + return a == b + } + + return true +} + +func typeCanBeUsedAs(t, as common.Type) bool { + nnT, okT := t.(*common.NonNull) + if okT { + t = nnT.OfType + } + + nnAs, okAs := as.(*common.NonNull) + if okAs { + as = nnAs.OfType + if !okT { + return false // nullable can not be used as non-null + } + } + + if t == as { + return true + } + + if lT, ok := t.(*common.List); ok { + if lAs, ok := as.(*common.List); ok { + return typeCanBeUsedAs(lT.OfType, lAs.OfType) + } + } + return false +} diff --git a/vendor/github.com/graph-gophers/graphql-go/introspection.go b/vendor/github.com/graph-gophers/graphql-go/introspection.go new file mode 100644 index 0000000000..7e515cf25f --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/introspection.go @@ -0,0 +1,117 @@ +package graphql + +import ( + "context" + "encoding/json" + + "github.com/graph-gophers/graphql-go/internal/exec/resolvable" + "github.com/graph-gophers/graphql-go/introspection" +) + +// Inspect allows inspection of the given schema. +func (s *Schema) Inspect() *introspection.Schema { + return introspection.WrapSchema(s.schema) +} + +// ToJSON encodes the schema in a JSON format used by tools like Relay. +func (s *Schema) ToJSON() ([]byte, error) { + result := s.exec(context.Background(), introspectionQuery, "", nil, &resolvable.Schema{ + Query: &resolvable.Object{}, + Schema: *s.schema, + }) + if len(result.Errors) != 0 { + panic(result.Errors[0]) + } + return json.MarshalIndent(result.Data, "", "\t") +} + +var introspectionQuery = ` + query { + __schema { + queryType { name } + mutationType { name } + subscriptionType { name } + types { + ...FullType + } + directives { + name + description + locations + args { + ...InputValue + } + } + } + } + fragment FullType on __Type { + kind + name + description + fields(includeDeprecated: true) { + name + description + args { + ...InputValue + } + type { + ...TypeRef + } + isDeprecated + deprecationReason + } + inputFields { + ...InputValue + } + interfaces { + ...TypeRef + } + enumValues(includeDeprecated: true) { + name + description + isDeprecated + deprecationReason + } + possibleTypes { + ...TypeRef + } + } + fragment InputValue on __InputValue { + name + description + type { ...TypeRef } + defaultValue + } + fragment TypeRef on __Type { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } +` diff --git a/vendor/github.com/graph-gophers/graphql-go/introspection/introspection.go b/vendor/github.com/graph-gophers/graphql-go/introspection/introspection.go new file mode 100644 index 0000000000..2f4acad0a6 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/introspection/introspection.go @@ -0,0 +1,313 @@ +package introspection + +import ( + "sort" + + "github.com/graph-gophers/graphql-go/internal/common" + "github.com/graph-gophers/graphql-go/internal/schema" +) + +type Schema struct { + schema *schema.Schema +} + +// WrapSchema is only used internally. +func WrapSchema(schema *schema.Schema) *Schema { + return &Schema{schema} +} + +func (r *Schema) Types() []*Type { + var names []string + for name := range r.schema.Types { + names = append(names, name) + } + sort.Strings(names) + + l := make([]*Type, len(names)) + for i, name := range names { + l[i] = &Type{r.schema.Types[name]} + } + return l +} + +func (r *Schema) Directives() []*Directive { + var names []string + for name := range r.schema.Directives { + names = append(names, name) + } + sort.Strings(names) + + l := make([]*Directive, len(names)) + for i, name := range names { + l[i] = &Directive{r.schema.Directives[name]} + } + return l +} + +func (r *Schema) QueryType() *Type { + t, ok := r.schema.EntryPoints["query"] + if !ok { + return nil + } + return &Type{t} +} + +func (r *Schema) MutationType() *Type { + t, ok := r.schema.EntryPoints["mutation"] + if !ok { + return nil + } + return &Type{t} +} + +func (r *Schema) SubscriptionType() *Type { + t, ok := r.schema.EntryPoints["subscription"] + if !ok { + return nil + } + return &Type{t} +} + +type Type struct { + typ common.Type +} + +// WrapType is only used internally. +func WrapType(typ common.Type) *Type { + return &Type{typ} +} + +func (r *Type) Kind() string { + return r.typ.Kind() +} + +func (r *Type) Name() *string { + if named, ok := r.typ.(schema.NamedType); ok { + name := named.TypeName() + return &name + } + return nil +} + +func (r *Type) Description() *string { + if named, ok := r.typ.(schema.NamedType); ok { + desc := named.Description() + if desc == "" { + return nil + } + return &desc + } + return nil +} + +func (r *Type) Fields(args *struct{ IncludeDeprecated bool }) *[]*Field { + var fields schema.FieldList + switch t := r.typ.(type) { + case *schema.Object: + fields = t.Fields + case *schema.Interface: + fields = t.Fields + default: + return nil + } + + var l []*Field + for _, f := range fields { + if d := f.Directives.Get("deprecated"); d == nil || args.IncludeDeprecated { + l = append(l, &Field{f}) + } + } + return &l +} + +func (r *Type) Interfaces() *[]*Type { + t, ok := r.typ.(*schema.Object) + if !ok { + return nil + } + + l := make([]*Type, len(t.Interfaces)) + for i, intf := range t.Interfaces { + l[i] = &Type{intf} + } + return &l +} + +func (r *Type) PossibleTypes() *[]*Type { + var possibleTypes []*schema.Object + switch t := r.typ.(type) { + case *schema.Interface: + possibleTypes = t.PossibleTypes + case *schema.Union: + possibleTypes = t.PossibleTypes + default: + return nil + } + + l := make([]*Type, len(possibleTypes)) + for i, intf := range possibleTypes { + l[i] = &Type{intf} + } + return &l +} + +func (r *Type) EnumValues(args *struct{ IncludeDeprecated bool }) *[]*EnumValue { + t, ok := r.typ.(*schema.Enum) + if !ok { + return nil + } + + var l []*EnumValue + for _, v := range t.Values { + if d := v.Directives.Get("deprecated"); d == nil || args.IncludeDeprecated { + l = append(l, &EnumValue{v}) + } + } + return &l +} + +func (r *Type) InputFields() *[]*InputValue { + t, ok := r.typ.(*schema.InputObject) + if !ok { + return nil + } + + l := make([]*InputValue, len(t.Values)) + for i, v := range t.Values { + l[i] = &InputValue{v} + } + return &l +} + +func (r *Type) OfType() *Type { + switch t := r.typ.(type) { + case *common.List: + return &Type{t.OfType} + case *common.NonNull: + return &Type{t.OfType} + default: + return nil + } +} + +type Field struct { + field *schema.Field +} + +func (r *Field) Name() string { + return r.field.Name +} + +func (r *Field) Description() *string { + if r.field.Desc == "" { + return nil + } + return &r.field.Desc +} + +func (r *Field) Args() []*InputValue { + l := make([]*InputValue, len(r.field.Args)) + for i, v := range r.field.Args { + l[i] = &InputValue{v} + } + return l +} + +func (r *Field) Type() *Type { + return &Type{r.field.Type} +} + +func (r *Field) IsDeprecated() bool { + return r.field.Directives.Get("deprecated") != nil +} + +func (r *Field) DeprecationReason() *string { + d := r.field.Directives.Get("deprecated") + if d == nil { + return nil + } + reason := d.Args.MustGet("reason").Value(nil).(string) + return &reason +} + +type InputValue struct { + value *common.InputValue +} + +func (r *InputValue) Name() string { + return r.value.Name.Name +} + +func (r *InputValue) Description() *string { + if r.value.Desc == "" { + return nil + } + return &r.value.Desc +} + +func (r *InputValue) Type() *Type { + return &Type{r.value.Type} +} + +func (r *InputValue) DefaultValue() *string { + if r.value.Default == nil { + return nil + } + s := r.value.Default.String() + return &s +} + +type EnumValue struct { + value *schema.EnumValue +} + +func (r *EnumValue) Name() string { + return r.value.Name +} + +func (r *EnumValue) Description() *string { + if r.value.Desc == "" { + return nil + } + return &r.value.Desc +} + +func (r *EnumValue) IsDeprecated() bool { + return r.value.Directives.Get("deprecated") != nil +} + +func (r *EnumValue) DeprecationReason() *string { + d := r.value.Directives.Get("deprecated") + if d == nil { + return nil + } + reason := d.Args.MustGet("reason").Value(nil).(string) + return &reason +} + +type Directive struct { + directive *schema.DirectiveDecl +} + +func (r *Directive) Name() string { + return r.directive.Name +} + +func (r *Directive) Description() *string { + if r.directive.Desc == "" { + return nil + } + return &r.directive.Desc +} + +func (r *Directive) Locations() []string { + return r.directive.Locs +} + +func (r *Directive) Args() []*InputValue { + l := make([]*InputValue, len(r.directive.Args)) + for i, v := range r.directive.Args { + l[i] = &InputValue{v} + } + return l +} diff --git a/vendor/github.com/graph-gophers/graphql-go/log/log.go b/vendor/github.com/graph-gophers/graphql-go/log/log.go new file mode 100644 index 0000000000..25569af7ca --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/log/log.go @@ -0,0 +1,23 @@ +package log + +import ( + "context" + "log" + "runtime" +) + +// Logger is the interface used to log panics that occur during query execution. It is settable via graphql.ParseSchema +type Logger interface { + LogPanic(ctx context.Context, value interface{}) +} + +// DefaultLogger is the default logger used to log panics that occur during query execution +type DefaultLogger struct{} + +// LogPanic is used to log recovered panic values that occur during query execution +func (l *DefaultLogger) LogPanic(_ context.Context, value interface{}) { + const size = 64 << 10 + buf := make([]byte, size) + buf = buf[:runtime.Stack(buf, false)] + log.Printf("graphql: panic occurred: %v\n%s", value, buf) +} diff --git a/vendor/github.com/graph-gophers/graphql-go/relay/relay.go b/vendor/github.com/graph-gophers/graphql-go/relay/relay.go new file mode 100644 index 0000000000..78e4dfdd51 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/relay/relay.go @@ -0,0 +1,70 @@ +package relay + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + graphql "github.com/graph-gophers/graphql-go" +) + +func MarshalID(kind string, spec interface{}) graphql.ID { + d, err := json.Marshal(spec) + if err != nil { + panic(fmt.Errorf("relay.MarshalID: %s", err)) + } + return graphql.ID(base64.URLEncoding.EncodeToString(append([]byte(kind+":"), d...))) +} + +func UnmarshalKind(id graphql.ID) string { + s, err := base64.URLEncoding.DecodeString(string(id)) + if err != nil { + return "" + } + i := strings.IndexByte(string(s), ':') + if i == -1 { + return "" + } + return string(s[:i]) +} + +func UnmarshalSpec(id graphql.ID, v interface{}) error { + s, err := base64.URLEncoding.DecodeString(string(id)) + if err != nil { + return err + } + i := strings.IndexByte(string(s), ':') + if i == -1 { + return errors.New("invalid graphql.ID") + } + return json.Unmarshal([]byte(s[i+1:]), v) +} + +type Handler struct { + Schema *graphql.Schema +} + +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var params struct { + Query string `json:"query"` + OperationName string `json:"operationName"` + Variables map[string]interface{} `json:"variables"` + } + if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + response := h.Schema.Exec(r.Context(), params.Query, params.OperationName, params.Variables) + responseJSON, err := json.Marshal(response) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Write(responseJSON) +} diff --git a/vendor/github.com/graph-gophers/graphql-go/time.go b/vendor/github.com/graph-gophers/graphql-go/time.go new file mode 100644 index 0000000000..829c502275 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/time.go @@ -0,0 +1,51 @@ +package graphql + +import ( + "encoding/json" + "fmt" + "time" +) + +// Time is a custom GraphQL type to represent an instant in time. It has to be added to a schema +// via "scalar Time" since it is not a predeclared GraphQL type like "ID". +type Time struct { + time.Time +} + +// ImplementsGraphQLType maps this custom Go type +// to the graphql scalar type in the schema. +func (Time) ImplementsGraphQLType(name string) bool { + return name == "Time" +} + +// UnmarshalGraphQL is a custom unmarshaler for Time +// +// This function will be called whenever you use the +// time scalar as an input +func (t *Time) UnmarshalGraphQL(input interface{}) error { + switch input := input.(type) { + case time.Time: + t.Time = input + return nil + case string: + var err error + t.Time, err = time.Parse(time.RFC3339, input) + return err + case int: + t.Time = time.Unix(int64(input), 0) + return nil + case float64: + t.Time = time.Unix(int64(input), 0) + return nil + default: + return fmt.Errorf("wrong type") + } +} + +// MarshalJSON is a custom marshaler for Time +// +// This function will be called whenever you +// query for fields that use the Time type +func (t Time) MarshalJSON() ([]byte, error) { + return json.Marshal(t.Time) +} diff --git a/vendor/github.com/graph-gophers/graphql-go/trace/trace.go b/vendor/github.com/graph-gophers/graphql-go/trace/trace.go new file mode 100644 index 0000000000..68b856ae71 --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/trace/trace.go @@ -0,0 +1,80 @@ +package trace + +import ( + "context" + "fmt" + + "github.com/graph-gophers/graphql-go/errors" + "github.com/graph-gophers/graphql-go/introspection" + opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go/ext" + "github.com/opentracing/opentracing-go/log" +) + +type TraceQueryFinishFunc func([]*errors.QueryError) +type TraceFieldFinishFunc func(*errors.QueryError) + +type Tracer interface { + TraceQuery(ctx context.Context, queryString string, operationName string, variables map[string]interface{}, varTypes map[string]*introspection.Type) (context.Context, TraceQueryFinishFunc) + TraceField(ctx context.Context, label, typeName, fieldName string, trivial bool, args map[string]interface{}) (context.Context, TraceFieldFinishFunc) +} + +type OpenTracingTracer struct{} + +func (OpenTracingTracer) TraceQuery(ctx context.Context, queryString string, operationName string, variables map[string]interface{}, varTypes map[string]*introspection.Type) (context.Context, TraceQueryFinishFunc) { + span, spanCtx := opentracing.StartSpanFromContext(ctx, "GraphQL request") + span.SetTag("graphql.query", queryString) + + if operationName != "" { + span.SetTag("graphql.operationName", operationName) + } + + if len(variables) != 0 { + span.LogFields(log.Object("graphql.variables", variables)) + } + + return spanCtx, func(errs []*errors.QueryError) { + if len(errs) > 0 { + msg := errs[0].Error() + if len(errs) > 1 { + msg += fmt.Sprintf(" (and %d more errors)", len(errs)-1) + } + ext.Error.Set(span, true) + span.SetTag("graphql.error", msg) + } + span.Finish() + } +} + +func (OpenTracingTracer) TraceField(ctx context.Context, label, typeName, fieldName string, trivial bool, args map[string]interface{}) (context.Context, TraceFieldFinishFunc) { + if trivial { + return ctx, noop + } + + span, spanCtx := opentracing.StartSpanFromContext(ctx, label) + span.SetTag("graphql.type", typeName) + span.SetTag("graphql.field", fieldName) + for name, value := range args { + span.SetTag("graphql.args."+name, value) + } + + return spanCtx, func(err *errors.QueryError) { + if err != nil { + ext.Error.Set(span, true) + span.SetTag("graphql.error", err.Error()) + } + span.Finish() + } +} + +func noop(*errors.QueryError) {} + +type NoopTracer struct{} + +func (NoopTracer) TraceQuery(ctx context.Context, queryString string, operationName string, variables map[string]interface{}, varTypes map[string]*introspection.Type) (context.Context, TraceQueryFinishFunc) { + return ctx, func(errs []*errors.QueryError) {} +} + +func (NoopTracer) TraceField(ctx context.Context, label, typeName, fieldName string, trivial bool, args map[string]interface{}) (context.Context, TraceFieldFinishFunc) { + return ctx, func(err *errors.QueryError) {} +} diff --git a/vendor/github.com/graph-gophers/graphql-go/trace/validation_trace.go b/vendor/github.com/graph-gophers/graphql-go/trace/validation_trace.go new file mode 100644 index 0000000000..e223f0fdbc --- /dev/null +++ b/vendor/github.com/graph-gophers/graphql-go/trace/validation_trace.go @@ -0,0 +1,17 @@ +package trace + +import ( + "github.com/graph-gophers/graphql-go/errors" +) + +type TraceValidationFinishFunc = TraceQueryFinishFunc + +type ValidationTracer interface { + TraceValidation() TraceValidationFinishFunc +} + +type NoopValidationTracer struct{} + +func (NoopValidationTracer) TraceValidation() TraceValidationFinishFunc { + return func(errs []*errors.QueryError) {} +}