mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-06-02 21:18:40 +00:00
cmd/devp2p: add --mode flag selecting crawl iterator
Wire the new discover.CrawlIterator into devp2p discv4/discv5 crawl behind a --mode flag (default 'lookup', i.e. existing behaviour). devp2p discv4 crawl --mode=fast --timeout 30s nodes.json devp2p discv5 crawl --mode=fast --timeout 30s nodes.json Smoke test against mainnet bootnodes for 30s on a residential link yields ~2.4x more nodes under --mode=fast (587 vs 240 in one run), with the new per-tick LogDist log showing a much more uniform distribution of query distances. Workers default to the existing --parallel value (16); pacing is RTT-driven. The 'lookup' default keeps existing behaviour byte-identical for any operator running the saved devp2p discv4 crawl from a script.
This commit is contained in:
parent
6c0d848d9c
commit
dfa3bbffae
2 changed files with 48 additions and 4 deletions
|
|
@ -91,7 +91,7 @@ var (
|
||||||
Name: "crawl",
|
Name: "crawl",
|
||||||
Usage: "Updates a nodes.json file with random nodes found in the DHT",
|
Usage: "Updates a nodes.json file with random nodes found in the DHT",
|
||||||
Action: discv4Crawl,
|
Action: discv4Crawl,
|
||||||
Flags: slices.Concat(discoveryNodeFlags, []cli.Flag{crawlTimeoutFlag, crawlParallelismFlag}),
|
Flags: slices.Concat(discoveryNodeFlags, []cli.Flag{crawlTimeoutFlag, crawlParallelismFlag, crawlModeFlag}),
|
||||||
}
|
}
|
||||||
discv4TestCommand = &cli.Command{
|
discv4TestCommand = &cli.Command{
|
||||||
Name: "test",
|
Name: "test",
|
||||||
|
|
@ -135,9 +135,14 @@ var (
|
||||||
}
|
}
|
||||||
crawlParallelismFlag = &cli.IntFlag{
|
crawlParallelismFlag = &cli.IntFlag{
|
||||||
Name: "parallel",
|
Name: "parallel",
|
||||||
Usage: "How many parallel discoveries to attempt.",
|
Usage: "How many parallel discoveries to attempt. Used both as the crawler harness's RequestENR worker count and (under -mode=fast) as the FINDNODE iterator's worker count.",
|
||||||
Value: 16,
|
Value: 16,
|
||||||
}
|
}
|
||||||
|
crawlModeFlag = &cli.StringFlag{
|
||||||
|
Name: "mode",
|
||||||
|
Usage: "Crawl iterator mode: 'lookup' (alpha-bounded Kademlia lookup) or 'fast' (one FINDNODE per peer with rotating prefix; sized by -parallel).",
|
||||||
|
Value: "lookup",
|
||||||
|
}
|
||||||
remoteEnodeFlag = &cli.StringFlag{
|
remoteEnodeFlag = &cli.StringFlag{
|
||||||
Name: "remote",
|
Name: "remote",
|
||||||
Usage: "Enode of the remote node under test",
|
Usage: "Enode of the remote node under test",
|
||||||
|
|
@ -259,7 +264,11 @@ func discv4Crawl(ctx *cli.Context) error {
|
||||||
disc, config := startV4(ctx)
|
disc, config := startV4(ctx)
|
||||||
defer disc.Close()
|
defer disc.Close()
|
||||||
|
|
||||||
c, err := newCrawler(inputSet, config.Bootnodes, disc, disc.RandomNodes())
|
iter, err := newDiscv4CrawlIterator(disc, config.Bootnodes, ctx.String(crawlModeFlag.Name), ctx.Int(crawlParallelismFlag.Name))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c, err := newCrawler(inputSet, config.Bootnodes, disc, iter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
@ -269,6 +278,20 @@ func discv4Crawl(ctx *cli.Context) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newDiscv4CrawlIterator(disc *discover.UDPv4, bootnodes []*enode.Node, mode string, parallel int) (enode.Iterator, error) {
|
||||||
|
switch mode {
|
||||||
|
case "", "lookup":
|
||||||
|
return disc.RandomNodes(), nil
|
||||||
|
case "fast":
|
||||||
|
return disc.CrawlIterator(discover.CrawlOptions{
|
||||||
|
Workers: parallel,
|
||||||
|
Seeds: bootnodes,
|
||||||
|
}), nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unknown -%s value %q (want 'lookup' or 'fast')", crawlModeFlag.Name, mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// discv4Test runs the protocol test suite.
|
// discv4Test runs the protocol test suite.
|
||||||
func discv4Test(ctx *cli.Context) error {
|
func discv4Test(ctx *cli.Context) error {
|
||||||
// Configure test package globals.
|
// Configure test package globals.
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ import (
|
||||||
"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v5test"
|
"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v5test"
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/p2p/discover"
|
"github.com/ethereum/go-ethereum/p2p/discover"
|
||||||
|
"github.com/ethereum/go-ethereum/p2p/enode"
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -58,6 +59,8 @@ var (
|
||||||
Action: discv5Crawl,
|
Action: discv5Crawl,
|
||||||
Flags: slices.Concat(discoveryNodeFlags, []cli.Flag{
|
Flags: slices.Concat(discoveryNodeFlags, []cli.Flag{
|
||||||
crawlTimeoutFlag,
|
crawlTimeoutFlag,
|
||||||
|
crawlParallelismFlag,
|
||||||
|
crawlModeFlag,
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
discv5TestCommand = &cli.Command{
|
discv5TestCommand = &cli.Command{
|
||||||
|
|
@ -111,7 +114,11 @@ func discv5Crawl(ctx *cli.Context) error {
|
||||||
disc, config := startV5(ctx)
|
disc, config := startV5(ctx)
|
||||||
defer disc.Close()
|
defer disc.Close()
|
||||||
|
|
||||||
c, err := newCrawler(inputSet, config.Bootnodes, disc, disc.RandomNodes())
|
iter, err := newDiscv5CrawlIterator(disc, config.Bootnodes, ctx.String(crawlModeFlag.Name), ctx.Int(crawlParallelismFlag.Name))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
c, err := newCrawler(inputSet, config.Bootnodes, disc, iter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
@ -121,6 +128,20 @@ func discv5Crawl(ctx *cli.Context) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newDiscv5CrawlIterator(disc *discover.UDPv5, bootnodes []*enode.Node, mode string, parallel int) (enode.Iterator, error) {
|
||||||
|
switch mode {
|
||||||
|
case "", "lookup":
|
||||||
|
return disc.RandomNodes(), nil
|
||||||
|
case "fast":
|
||||||
|
return disc.CrawlIterator(discover.CrawlOptions{
|
||||||
|
Workers: parallel,
|
||||||
|
Seeds: bootnodes,
|
||||||
|
}), nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unknown -%s value %q (want 'lookup' or 'fast')", crawlModeFlag.Name, mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// discv5Test runs the protocol test suite.
|
// discv5Test runs the protocol test suite.
|
||||||
func discv5Test(ctx *cli.Context) error {
|
func discv5Test(ctx *cli.Context) error {
|
||||||
suite := &v5test.Suite{
|
suite := &v5test.Suite{
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue