diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 17bf438e2a..a331abc9fd 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -93,6 +93,8 @@ var ( utils.ExitWhenSyncedFlag, utils.GCModeFlag, utils.LightServFlag, + utils.LightBandwidthInFlag, + utils.LightBandwidthOutFlag, utils.LightPeersFlag, utils.LightKDFFlag, utils.WhitelistFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index a262037168..0338e447e4 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -81,6 +81,8 @@ var AppHelpFlagGroups = []flagGroup{ utils.EthStatsURLFlag, utils.IdentityFlag, utils.LightServFlag, + utils.LightBandwidthInFlag, + utils.LightBandwidthOutFlag, utils.LightPeersFlag, utils.LightKDFFlag, utils.WhitelistFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5b8ebb481f..4db59097d7 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -199,9 +199,19 @@ var ( } LightServFlag = cli.IntFlag{ Name: "lightserv", - Usage: "Maximum percentage of time allowed for serving LES requests (0-90)", + Usage: "Maximum percentage of time allowed for serving LES requests (multi-threaded processing allows values over 100)", Value: 0, } + LightBandwidthInFlag = cli.IntFlag{ + Name: "lightbwin", + Usage: "Incoming bandwidth limit for light server (1000 bytes/sec, 0 = unlimited)", + Value: 1000, + } + LightBandwidthOutFlag = cli.IntFlag{ + Name: "lightbwout", + Usage: "Outgoing bandwidth limit for light server (1000 bytes/sec, 0 = unlimited)", + Value: 5000, + } LightPeersFlag = cli.IntFlag{ Name: "lightpeers", Usage: "Maximum number of LES client peers", @@ -1305,6 +1315,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(LightServFlag.Name) { cfg.LightServ = ctx.GlobalInt(LightServFlag.Name) } + cfg.LightBandwidthIn = ctx.GlobalInt(LightBandwidthInFlag.Name) + cfg.LightBandwidthOut = ctx.GlobalInt(LightBandwidthOutFlag.Name) if ctx.GlobalIsSet(LightPeersFlag.Name) { cfg.LightPeers = ctx.GlobalInt(LightPeersFlag.Name) } diff --git a/core/blockchain.go b/core/blockchain.go index d6dad27991..1252454e3e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -134,10 +134,11 @@ type BlockChain struct { procInterrupt int32 // interrupt signaler for block processing wg sync.WaitGroup // chain processing wait group for shutting down - engine consensus.Engine - processor Processor // block processor interface - validator Validator // block and state validator interface - vmConfig vm.Config + engine consensus.Engine + processor Processor // block processor interface + validator Validator // block and state validator interface + vmConfig vm.Config + procFeedback chan bool badBlocks *lru.Cache // Bad block cache shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. @@ -370,6 +371,14 @@ func (bc *BlockChain) CurrentFastBlock() *types.Block { return bc.currentFastBlock.Load().(*types.Block) } +// SetProcFeedback adds a feedback channel where true is sent each time block +// processing begins and false is sent when it is finished. +func (bc *BlockChain) SetProcFeedback(procFeedback chan bool) { + bc.procmu.Lock() + defer bc.procmu.Unlock() + bc.procFeedback = procFeedback +} + // SetProcessor sets the processor required for making state modifications. func (bc *BlockChain) SetProcessor(processor Processor) { bc.procmu.Lock() @@ -1090,6 +1099,25 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { if len(chain) == 0 { return 0, nil } + + // send block processing feedback if needed + bc.procmu.RLock() + procFeedback := bc.procFeedback + bc.procmu.RUnlock() + + if procFeedback != nil { + select { + case procFeedback <- true: + default: + } + defer func() { + select { + case procFeedback <- false: + default: + } + }() + } + // Remove already known canon-blocks var ( block, prev *types.Block diff --git a/eth/backend.go b/eth/backend.go index 6710e45131..cccb5993f1 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -54,6 +54,7 @@ import ( type LesServer interface { Start(srvr *p2p.Server) Stop() + APIs() []rpc.API Protocols() []p2p.Protocol SetBloomBitsIndexer(bbIndexer *core.ChainIndexer) } @@ -267,6 +268,10 @@ func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainCo func (s *Ethereum) APIs() []rpc.API { apis := ethapi.GetAPIs(s.APIBackend) + // Append any APIs exposed explicitly by the les server + if s.lesServer != nil { + apis = append(apis, s.lesServer.APIs()...) + } // Append any APIs exposed explicitly by the consensus engine apis = append(apis, s.engine.APIs(s.BlockChain())...) diff --git a/eth/config.go b/eth/config.go index aca9b5e68b..740e6825b0 100644 --- a/eth/config.go +++ b/eth/config.go @@ -98,9 +98,11 @@ type Config struct { Whitelist map[uint64]common.Hash `toml:"-"` // Light client options - LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests - LightPeers int `toml:",omitempty"` // Maximum number of LES client peers - OnlyAnnounce bool // Maximum number of LES client peers + LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests + LightBandwidthIn int `toml:",omitempty"` // Incoming bandwidth limit for light servers + LightBandwidthOut int `toml:",omitempty"` // Outgoing bandwidth limit for light servers + LightPeers int `toml:",omitempty"` // Maximum number of LES client peers + OnlyAnnounce bool // Maximum number of LES client peers // Ultra Light client options ULC *ULCConfig `toml:",omitempty"` diff --git a/eth/gen_config.go b/eth/gen_config.go index e05b963abe..30ff8b6e13 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -24,6 +24,8 @@ func (c Config) MarshalTOML() (interface{}, error) { SyncMode downloader.SyncMode NoPruning bool LightServ int `toml:",omitempty"` + LightBandwidthIn int `toml:",omitempty"` + LightBandwidthOut int `toml:",omitempty"` LightPeers int `toml:",omitempty"` OnlyAnnounce bool ULC *ULCConfig `toml:",omitempty"` @@ -55,6 +57,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.SyncMode = c.SyncMode enc.NoPruning = c.NoPruning enc.LightServ = c.LightServ + enc.LightBandwidthIn = c.LightBandwidthIn + enc.LightBandwidthOut = c.LightBandwidthOut enc.LightPeers = c.LightPeers enc.OnlyAnnounce = c.OnlyAnnounce enc.ULC = c.ULC @@ -91,6 +95,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { SyncMode *downloader.SyncMode NoPruning *bool LightServ *int `toml:",omitempty"` + LightBandwidthIn *int `toml:",omitempty"` + LightBandwidthOut *int `toml:",omitempty"` LightPeers *int `toml:",omitempty"` OnlyAnnounce *bool ULC *ULCConfig `toml:",omitempty"` @@ -135,6 +141,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.LightServ != nil { c.LightServ = *dec.LightServ } + if dec.LightBandwidthIn != nil { + c.LightBandwidthIn = *dec.LightBandwidthIn + } + if dec.LightBandwidthOut != nil { + c.LightBandwidthOut = *dec.LightBandwidthOut + } if dec.LightPeers != nil { c.LightPeers = *dec.LightPeers } diff --git a/les/api.go b/les/api.go new file mode 100644 index 0000000000..97ed7a41e6 --- /dev/null +++ b/les/api.go @@ -0,0 +1,176 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package les + +import ( + "errors" + "sync" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/p2p/enode" +) + +var ( + ErrMinBW = errors.New("bandwidth too small") + ErrTotalBW = errors.New("total bandwidth exceeded") +) + +// PublicLesServerAPI provides an API to access the les server. +// It offers only methods that operate on public data that is freely available to anyone. +type PrivateLesServerAPI struct { + server *LesServer + pm *ProtocolManager + vip *vipClientPool +} + +// NewPublicLesServerAPI creates a new les server API. +func NewPrivateLesServerAPI(server *LesServer) *PrivateLesServerAPI { + vip := &vipClientPool{ + clients: make(map[enode.ID]vipClientInfo), + totalBw: server.totalBandwidth, + pm: server.protocolManager, + } + server.protocolManager.vipClientPool = vip + return &PrivateLesServerAPI{ + server: server, + pm: server.protocolManager, + vip: vip, + } +} + +// TotalBandwidth queries total available bandwidth for all clients +func (api *PrivateLesServerAPI) TotalBandwidth() hexutil.Uint64 { + return hexutil.Uint64(api.server.totalBandwidth) +} + +// MinimumBandwidth queries minimum assignable bandwidth for a single client +func (api *PrivateLesServerAPI) MinimumBandwidth() hexutil.Uint64 { + return hexutil.Uint64(api.server.minBandwidth) +} + +// vipClientPool stores information about prioritized clients +type vipClientPool struct { + lock sync.Mutex + pm *ProtocolManager + clients map[enode.ID]vipClientInfo + totalBw, totalVipBw, totalConnectedBw uint64 + vipCount int +} + +// vipClientInfo entries exist for all prioritized clients and currently connected free clients +type vipClientInfo struct { + bw uint64 // zero for non-vip clients + connected bool + updateBw func(uint64) +} + +// SetClientBandwidth sets the priority bandwidth assigned to a given client. +// If the assigned bandwidth is bigger than zero then connection is always +// guaranteed. The sum of bandwidth assigned to priority clients can not exceed +// the total available bandwidth. +// +// Note: assigned bandwidth can be changed while the client is connected with +// immediate effect. +func (api *PrivateLesServerAPI) SetClientBandwidth(id enode.ID, bw uint64) error { + if bw != 0 && bw < api.server.minBandwidth { + return ErrMinBW + } + + api.vip.lock.Lock() + defer api.vip.lock.Unlock() + + c := api.vip.clients[id] + if api.vip.totalVipBw+bw > api.vip.totalBw+c.bw { + return ErrTotalBW + } + api.vip.totalVipBw += bw - c.bw + if c.updateBw != nil && bw != 0 { + c.updateBw(bw) + } + if c.connected { + if c.bw != 0 { + api.vip.vipCount-- + } + if bw != 0 { + api.vip.vipCount++ + } + api.vip.totalConnectedBw += bw - c.bw + api.pm.clientPool.setConnLimit(api.pm.maxFreePeers(api.vip.vipCount, api.vip.totalConnectedBw)) + } + if c.updateBw != nil && bw == 0 { + c.updateBw(bw) + } + if bw != 0 || c.connected { + c.bw = bw + api.vip.clients[id] = c + } else { + delete(api.vip.clients, id) + } + return nil +} + +// GetClientBandwidth returns the bandwidth assigned to a given client +func (api *PrivateLesServerAPI) GetClientBandwidth(id enode.ID) hexutil.Uint64 { + api.vip.lock.Lock() + defer api.vip.lock.Unlock() + + return hexutil.Uint64(api.vip.clients[id].bw) +} + +// connect should be called when a new client is connected. The callback function +// is called when the assigned bandwidth is changed while the client is connected. +// It returns the priority bandwidth or zero if the client is not prioritized. +// It also returns whether the client can be accepted. +// +// Note: vipClientPool also stores a record about free clients while they are +// connected in order to be able to assign priority to them later with the callback +// function if necessary. +func (v *vipClientPool) connect(id enode.ID, updateBw func(uint64)) (uint64, bool) { + v.lock.Lock() + defer v.lock.Unlock() + + c := v.clients[id] + if c.connected { + return 0, false + } + c.connected = true + c.updateBw = updateBw + v.clients[id] = c + if c.bw != 0 { + v.vipCount++ + } + v.totalConnectedBw += c.bw + v.pm.clientPool.setConnLimit(v.pm.maxFreePeers(v.vipCount, v.totalConnectedBw)) + return c.bw, true +} + +// disconnect should be called when a client is disconnected. +// It should be called for all clients accepted by connect even if not prioritized. +func (v *vipClientPool) disconnect(id enode.ID) { + v.lock.Lock() + defer v.lock.Unlock() + + c := v.clients[id] + c.connected = false + if c.bw != 0 { + v.clients[id] = c + v.vipCount-- + } else { + delete(v.clients, id) + } + v.totalConnectedBw -= c.bw + v.pm.clientPool.setConnLimit(v.pm.maxFreePeers(v.vipCount, v.totalConnectedBw)) +} diff --git a/les/backend.go b/les/backend.go index cd99f8f813..67ddf17e4c 100644 --- a/les/backend.go +++ b/les/backend.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" @@ -100,7 +101,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { chainConfig: chainConfig, eventMux: ctx.EventMux, peers: peers, - reqDist: newRequestDistributor(peers, quitSync), + reqDist: newRequestDistributor(peers, quitSync, &mclock.System{}), accountManager: ctx.AccountManager, engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, false, chainDb), shutdownChan: make(chan bool), diff --git a/les/bandwidth_api_test.go b/les/bandwidth_api_test.go new file mode 100644 index 0000000000..d7b343d9bb --- /dev/null +++ b/les/bandwidth_api_test.go @@ -0,0 +1,475 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "context" + "errors" + "flag" + "fmt" + "io/ioutil" + "math/rand" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/les/flowcontrol" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rpc" + colorable "github.com/mattn/go-colorable" +) + +/* +This test is not meant to be a part of the automatic testing process because it +runs for a long time and also requires a large database in order to do a meaningful +request performance test. When testServerDataDir is empty, the test is skipped. +*/ + +const ( + testServerDataDir = "" // should always be empty on the master branch + testServerBandwidth = 200 + testMaxClients = 10 + testTolerance = 0.1 + minRelBw = 0.2 +) + +func TestBandwidthAPI3(t *testing.T) { + testBandwidthAPI(t, 3) +} + +func TestBandwidthAPI6(t *testing.T) { + testBandwidthAPI(t, 6) +} + +func TestBandwidthAPI10(t *testing.T) { + testBandwidthAPI(t, 10) +} + +// testBandwidthAPI runs an end-to-end simulation test connecting one server with +// a given number of clients. It sets different priority bandwidths to all clients +// except a randomly selected one which runs in free client mode. All clients send +// similar requests at the maximum allowed rate and the test verifies whether the +// ratio of processed requests is close enough to the ratio of assigned bandwidths. +// Running multiple rounds with different settings ensures that changing bandwidth +// while connected and going back and forth between free and priority mode with +// the supplied API calls is also thoroughly tested. +func testBandwidthAPI(t *testing.T, clientCount int) { + if testServerDataDir == "" { + // Skip test if no data dir specified + return + } + + testSim(t, 1, clientCount, []string{testServerDataDir}, nil, func(ctx context.Context, net *simulations.Network, servers []*simulations.Node, clients []*simulations.Node) { + if len(servers) != 1 { + t.Fatalf("Invalid number of servers: %d", len(servers)) + } + server := servers[0] + + clientRpcClients := make([]*rpc.Client, len(clients)) + + serverRpcClient, err := server.Client() + if err != nil { + t.Fatalf("Failed to obtain rpc client: %v", err) + } + headNum, headHash := getHead(ctx, t, serverRpcClient) + totalBw, minBw := bandwidthLimits(ctx, t, serverRpcClient) + fmt.Printf("Server totalBw: %d minBw: %d head number: %d head hash: %064x\n", totalBw, minBw, headNum, headHash) + reqMinBw := uint64(float64(totalBw) * minRelBw / (minRelBw + float64(len(clients)-1))) + if minBw > reqMinBw { + t.Fatalf("Minimum client bandwidth (%d) bigger than required minimum for this test (%d)", minBw, reqMinBw) + } + + freeIdx := rand.Intn(len(clients)) + freeBw := totalBw / testMaxClients + + for i, client := range clients { + var err error + clientRpcClients[i], err = client.Client() + if err != nil { + t.Fatalf("Failed to obtain rpc client: %v", err) + } + + fmt.Println("connecting client", i) + if i != freeIdx { + setBandwidth(ctx, t, serverRpcClient, client.ID(), totalBw/uint64(len(clients))) + } + net.Connect(client.ID(), server.ID()) + + for { + select { + case <-ctx.Done(): + t.Fatalf("Timeout") + default: + } + num, hash := getHead(ctx, t, clientRpcClients[i]) + if num == headNum && hash == headHash { + fmt.Println("client", i, "synced") + break + } + time.Sleep(time.Millisecond * 200) + } + } + + var wg sync.WaitGroup + stop := make(chan struct{}) + + reqCount := make([]uint64, len(clientRpcClients)) + for i, c := range clientRpcClients { + wg.Add(1) + i, c := i, c + go func() { + queue := make(chan struct{}, 100) + var count uint64 + for { + select { + case queue <- struct{}{}: + wg.Add(1) + go func() { + testRequest(ctx, t, c) + wg.Done() + <-queue + count++ + atomic.StoreUint64(&reqCount[i], count) + }() + case <-stop: + wg.Done() + return + case <-ctx.Done(): + wg.Done() + return + } + } + }() + } + + processedSince := func(start []uint64) []uint64 { + res := make([]uint64, len(reqCount)) + for i, _ := range reqCount { + res[i] = atomic.LoadUint64(&reqCount[i]) + if start != nil { + res[i] -= start[i] + } + } + return res + } + + weights := make([]float64, len(clients)) + for c := 0; c < 5; c++ { + setBandwidth(ctx, t, serverRpcClient, clients[freeIdx].ID(), freeBw) + freeIdx = rand.Intn(len(clients)) + var sum float64 + for i, _ := range clients { + if i == freeIdx { + weights[i] = 0 + } else { + weights[i] = rand.Float64()*(1-minRelBw) + minRelBw + } + sum += weights[i] + } + for i, client := range clients { + weights[i] *= float64(totalBw-freeBw-100) / sum + bandwidth := uint64(weights[i]) + if i != freeIdx && bandwidth < getBandwidth(ctx, t, serverRpcClient, client.ID()) { + setBandwidth(ctx, t, serverRpcClient, client.ID(), bandwidth) + } + } + setBandwidth(ctx, t, serverRpcClient, clients[freeIdx].ID(), 0) + for i, client := range clients { + bandwidth := uint64(weights[i]) + if i != freeIdx && bandwidth > getBandwidth(ctx, t, serverRpcClient, client.ID()) { + setBandwidth(ctx, t, serverRpcClient, client.ID(), bandwidth) + } + } + weights[freeIdx] = float64(freeBw) + for i, _ := range clients { + weights[i] /= float64(totalBw) + } + + time.Sleep(flowcontrol.DecParamDelay) + fmt.Println("starting measurement") + start := processedSince(nil) + for { + select { + case <-ctx.Done(): + t.Fatalf("Timeout") + default: + } + + processed := processedSince(start) + var avg uint64 + fmt.Printf("Processed") + for i, p := range processed { + fmt.Printf(" %d", p) + processed[i] = uint64(float64(p) / weights[i]) + avg += processed[i] + } + avg /= uint64(len(processed)) + + if avg >= 10000 { + var maxDev float64 + for _, p := range processed { + dev := float64(int64(p-avg)) / float64(avg) + fmt.Printf(" %7.4f", dev) + if dev < 0 { + dev = -dev + } + if dev > maxDev { + maxDev = dev + } + } + fmt.Printf(" max deviation: %f\n", maxDev) + if maxDev <= testTolerance { + fmt.Println("success") + break + } + } else { + fmt.Println() + } + time.Sleep(time.Millisecond * 200) + } + } + + close(stop) + wg.Wait() + + for i, count := range reqCount { + fmt.Println("client", i, "processed", count) + } + }) +} + +func getHead(ctx context.Context, t *testing.T, client *rpc.Client) (uint64, common.Hash) { + res := make(map[string]interface{}) + if err := client.CallContext(ctx, &res, "eth_getBlockByNumber", "latest", false); err != nil { + t.Fatalf("Failed to obtain head block: %v", err) + } + numStr, ok := res["number"].(string) + if !ok { + t.Fatalf("RPC block number field invalid") + } + num, err := hexutil.DecodeUint64(numStr) + if err != nil { + t.Fatalf("Failed to decode RPC block number: %v", err) + } + hashStr, ok := res["hash"].(string) + if !ok { + t.Fatalf("RPC block number field invalid") + } + hash := common.HexToHash(hashStr) + return num, hash +} + +func testRequest(ctx context.Context, t *testing.T, client *rpc.Client) { + //res := make(map[string]interface{}) + var res string + var addr common.Address + rand.Read(addr[:]) + // if err := client.CallContext(ctx, &res, "eth_getProof", addr, nil, "latest"); err != nil { + if err := client.CallContext(ctx, &res, "eth_getBalance", addr, "latest"); err != nil { + t.Fatalf("Failed to obtain Merkle proof: %v", err) + } +} + +func setBandwidth(ctx context.Context, t *testing.T, server *rpc.Client, clientID enode.ID, bw uint64) { + if err := server.CallContext(ctx, nil, "les_setClientBandwidth", clientID, bw); err != nil { + t.Fatalf("Failed to set client bandwidth: %v", err) + } +} + +func getBandwidth(ctx context.Context, t *testing.T, server *rpc.Client, clientID enode.ID) uint64 { + var s string + if err := server.CallContext(ctx, &s, "les_getClientBandwidth", clientID); err != nil { + t.Fatalf("Failed to get client bandwidth: %v", err) + } + bw, err := hexutil.DecodeUint64(s) + if err != nil { + t.Fatalf("Failed to decode client bandwidth: %v", err) + } + return bw +} + +func bandwidthLimits(ctx context.Context, t *testing.T, server *rpc.Client) (uint64, uint64) { + var s string + if err := server.CallContext(ctx, &s, "les_totalBandwidth"); err != nil { + t.Fatalf("Failed to query total bandwidth: %v", err) + } + total, err := hexutil.DecodeUint64(s) + if err != nil { + t.Fatalf("Failed to decode total bandwidth: %v", err) + } + if err := server.CallContext(ctx, &s, "les_minimumBandwidth"); err != nil { + t.Fatalf("Failed to query minimum bandwidth: %v", err) + } + min, err := hexutil.DecodeUint64(s) + if err != nil { + t.Fatalf("Failed to decode minimum bandwidth: %v", err) + } + return total, min +} + +func init() { + flag.Parse() + // register the Delivery service which will run as a devp2p + // protocol when using the exec adapter + adapters.RegisterServices(services) + + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) +} + +var ( + adapter = flag.String("adapter", "exec", "type of simulation: sim|socket|exec|docker") + loglevel = flag.Int("loglevel", 0, "verbosity of logs") + nodes = flag.Int("nodes", 0, "number of nodes") +) + +var services = adapters.Services{ + "lesclient": newLesClientService, + "lesserver": newLesServerService, +} + +func NewNetwork() (*simulations.Network, func(), error) { + adapter, adapterTeardown, err := NewAdapter(*adapter, services) + if err != nil { + return nil, adapterTeardown, err + } + defaultService := "streamer" + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ + ID: "0", + DefaultService: defaultService, + }) + teardown := func() { + adapterTeardown() + net.Shutdown() + } + + return net, teardown, nil +} + +func NewAdapter(adapterType string, services adapters.Services) (adapter adapters.NodeAdapter, teardown func(), err error) { + teardown = func() {} + switch adapterType { + case "sim": + adapter = adapters.NewSimAdapter(services) + // case "socket": + // adapter = adapters.NewSocketAdapter(services) + case "exec": + baseDir, err0 := ioutil.TempDir("", "les-test") + if err0 != nil { + return nil, teardown, err0 + } + teardown = func() { os.RemoveAll(baseDir) } + adapter = adapters.NewExecAdapter(baseDir) + /*case "docker": + adapter, err = adapters.NewDockerAdapter() + if err != nil { + return nil, teardown, err + }*/ + default: + return nil, teardown, errors.New("adapter needs to be one of sim, socket, exec, docker") + } + return adapter, teardown, nil +} + +func testSim(t *testing.T, serverCount, clientCount int, serverDir, clientDir []string, test func(ctx context.Context, net *simulations.Network, servers []*simulations.Node, clients []*simulations.Node)) { + net, teardown, err := NewNetwork() + defer teardown() + if err != nil { + t.Fatalf("Failed to create network: %v", err) + } + timeout := 1800 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + servers := make([]*simulations.Node, serverCount) + clients := make([]*simulations.Node, clientCount) + + for i, _ := range clients { + clientconf := adapters.RandomNodeConfig() + clientconf.Services = []string{"lesclient"} + if len(clientDir) == clientCount { + clientconf.DataDir = clientDir[i] + } + client, err := net.NewNodeWithConfig(clientconf) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + clients[i] = client + } + + for i, _ := range servers { + serverconf := adapters.RandomNodeConfig() + serverconf.Services = []string{"lesserver"} + if len(serverDir) == serverCount { + serverconf.DataDir = serverDir[i] + } + server, err := net.NewNodeWithConfig(serverconf) + if err != nil { + t.Fatalf("Failed to create server: %v", err) + } + servers[i] = server + } + + for _, client := range clients { + if err := net.Start(client.ID()); err != nil { + t.Fatalf("Failed to start client node: %v", err) + } + } + for _, server := range servers { + if err := net.Start(server.ID()); err != nil { + t.Fatalf("Failed to start server node: %v", err) + } + } + + test(ctx, net, servers, clients) +} + +func newLesClientService(ctx *adapters.ServiceContext) (node.Service, error) { + config := eth.DefaultConfig + config.SyncMode = downloader.LightSync + config.Ethash.PowMode = ethash.ModeFake + return New(ctx.NodeContext, &config) +} + +func newLesServerService(ctx *adapters.ServiceContext) (node.Service, error) { + config := eth.DefaultConfig + config.SyncMode = downloader.FullSync + config.LightServ = testServerBandwidth + config.LightPeers = testMaxClients + ethereum, err := eth.New(ctx.NodeContext, &config) + if err != nil { + return nil, err + } + + server, err := NewLesServer(ethereum, &config) + if err != nil { + return nil, err + } + ethereum.AddLesServer(server) + return ethereum, nil +} diff --git a/les/benchmark.go b/les/benchmark.go new file mode 100644 index 0000000000..9340b22e37 --- /dev/null +++ b/les/benchmark.go @@ -0,0 +1,632 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package les + +import ( + "encoding/binary" + "fmt" + "math/big" + "math/rand" + "sort" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/les/flowcontrol" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" +) + +// requestBenchmark is an interface for different randomized request generators +type requestBenchmark interface { + // init initializes the generator for generating the given number of randomized requests + init(pm *ProtocolManager, count int) error + // request initiates sending a single request to the given peer + request(peer *peer, index int) error +} + +type benchmarkBlockHeaders struct { + amount, skip int + reverse, byHash bool + offset, randMax int64 + hashes []common.Hash +} + +func (b *benchmarkBlockHeaders) init(pm *ProtocolManager, count int) error { + d := int64(b.amount-1) * int64(b.skip+1) + b.offset = 0 + b.randMax = pm.blockchain.CurrentHeader().Number.Int64() + 1 - d + if b.randMax < 0 { + return fmt.Errorf("chain is too short") + } + if b.reverse { + b.offset = d + } + if b.byHash { + b.hashes = make([]common.Hash, count) + for i, _ := range b.hashes { + b.hashes[i] = rawdb.ReadCanonicalHash(pm.chainDb, uint64(b.offset+rand.Int63n(b.randMax))) + } + } + return nil +} + +func (b *benchmarkBlockHeaders) request(peer *peer, index int) error { + if b.byHash { + return peer.RequestHeadersByHash(0, 0, b.hashes[index], b.amount, b.skip, b.reverse) + } else { + return peer.RequestHeadersByNumber(0, 0, uint64(b.offset+rand.Int63n(b.randMax)), b.amount, b.skip, b.reverse) + } +} + +type benchmarkBodiesOrReceipts struct { + receipts bool + hashes []common.Hash +} + +func (b *benchmarkBodiesOrReceipts) init(pm *ProtocolManager, count int) error { + randMax := pm.blockchain.CurrentHeader().Number.Int64() + 1 + b.hashes = make([]common.Hash, count) + for i, _ := range b.hashes { + b.hashes[i] = rawdb.ReadCanonicalHash(pm.chainDb, uint64(rand.Int63n(randMax))) + } + return nil +} + +func (b *benchmarkBodiesOrReceipts) request(peer *peer, index int) error { + if b.receipts { + return peer.RequestReceipts(0, 0, []common.Hash{b.hashes[index]}) + } else { + return peer.RequestBodies(0, 0, []common.Hash{b.hashes[index]}) + } +} + +type benchmarkProofsOrCode struct { + code bool + headHash common.Hash +} + +func (b *benchmarkProofsOrCode) init(pm *ProtocolManager, count int) error { + b.headHash = pm.blockchain.CurrentHeader().Hash() + return nil +} + +func (b *benchmarkProofsOrCode) request(peer *peer, index int) error { + key := make([]byte, 32) + rand.Read(key) + if b.code { + return peer.RequestCode(0, 0, []CodeReq{CodeReq{BHash: b.headHash, AccKey: key}}) + } else { + return peer.RequestProofs(0, 0, []ProofReq{ProofReq{BHash: b.headHash, Key: key}}) + } +} + +type benchmarkHelperTrie struct { + bloom bool + reqCount int + sectionCount, headNum uint64 +} + +func (b *benchmarkHelperTrie) init(pm *ProtocolManager, count int) error { + if b.bloom { + b.sectionCount, b.headNum, _ = pm.server.bloomTrieIndexer.Sections() + } else { + b.sectionCount, _, _ = pm.server.chtIndexer.Sections() + b.sectionCount /= (params.CHTFrequencyClient / params.CHTFrequencyServer) + b.headNum = b.sectionCount*params.CHTFrequencyClient - 1 + } + if b.sectionCount == 0 { + return fmt.Errorf("no processed sections available") + } + return nil +} + +func (b *benchmarkHelperTrie) request(peer *peer, index int) error { + reqs := make([]HelperTrieReq, b.reqCount) + + if b.bloom { + bitIdx := uint16(rand.Intn(2048)) + for i, _ := range reqs { + key := make([]byte, 10) + binary.BigEndian.PutUint16(key[:2], bitIdx) + binary.BigEndian.PutUint64(key[2:], uint64(rand.Int63n(int64(b.sectionCount)))) + reqs[i] = HelperTrieReq{Type: htBloomBits, TrieIdx: b.sectionCount - 1, Key: key} + } + } else { + for i, _ := range reqs { + key := make([]byte, 8) + binary.BigEndian.PutUint64(key[:], uint64(rand.Int63n(int64(b.headNum)))) + reqs[i] = HelperTrieReq{Type: htCanonical, TrieIdx: b.sectionCount - 1, Key: key, AuxReq: auxHeader} + } + } + + return peer.RequestHelperTrieProofs(0, 0, reqs) +} + +type benchmarkTxSend struct { + txs types.Transactions +} + +func (b *benchmarkTxSend) init(pm *ProtocolManager, count int) error { + key, _ := crypto.GenerateKey() + addr := crypto.PubkeyToAddress(key.PublicKey) + signer := types.NewEIP155Signer(big.NewInt(18)) + b.txs = make(types.Transactions, count) + + for i, _ := range b.txs { + data := make([]byte, txSizeCostLimit) + rand.Read(data) + tx, err := types.SignTx(types.NewTransaction(0, addr, new(big.Int), 0, new(big.Int), data), signer, key) + if err != nil { + panic(err) + } + b.txs[i] = tx + } + return nil +} + +func (b *benchmarkTxSend) request(peer *peer, index int) error { + enc, _ := rlp.EncodeToBytes(types.Transactions{b.txs[index]}) + return peer.SendTxs(0, 0, enc) +} + +type benchmarkTxStatus struct{} + +func (b *benchmarkTxStatus) init(pm *ProtocolManager, count int) error { + return nil +} + +func (b *benchmarkTxStatus) request(peer *peer, index int) error { + var hash common.Hash + rand.Read(hash[:]) + return peer.RequestTxStatus(0, 0, []common.Hash{hash}) +} + +type benchmarkType struct { + name string + newInstance func() requestBenchmark + outSizeCorr uint32 + avgTimeCorr float64 +} + +// benchmarkTypes describes different benchmark scenarios +var benchmarkTypes = map[string]benchmarkType{ + "header1n": {name: "header by number (single)", newInstance: func() requestBenchmark { + return &benchmarkBlockHeaders{amount: 1} + }}, + "header1h": {name: "header by hash (single)", newInstance: func() requestBenchmark { + return &benchmarkBlockHeaders{amount: 1, byHash: true} + }}, + "header192n": {name: "headers by number (192)", newInstance: func() requestBenchmark { + return &benchmarkBlockHeaders{amount: 192} + }}, + "header192hr": {name: "headers by hash (192, reverse)", newInstance: func() requestBenchmark { + return &benchmarkBlockHeaders{amount: 192, byHash: true, reverse: true} + }}, + "body": {name: "block body", newInstance: func() requestBenchmark { + return &benchmarkBodiesOrReceipts{receipts: false} + }}, + "receipts": {name: "block receipts", newInstance: func() requestBenchmark { + return &benchmarkBodiesOrReceipts{receipts: true} + }}, + "proof": {name: "merkle proof", newInstance: func() requestBenchmark { + return &benchmarkProofsOrCode{code: false} + }, outSizeCorr: 500, avgTimeCorr: 2.5}, + "code": {name: "contract code", newInstance: func() requestBenchmark { + return &benchmarkProofsOrCode{code: true} + }, outSizeCorr: 100000, avgTimeCorr: 1.5}, + "cht1": {name: "cht (single)", newInstance: func() requestBenchmark { + return &benchmarkHelperTrie{bloom: false, reqCount: 1} + }}, + "cht16": {name: "cht (16)", newInstance: func() requestBenchmark { + return &benchmarkHelperTrie{bloom: false, reqCount: 16} + }}, + "bloom1": {name: "bloom trie (single)", newInstance: func() requestBenchmark { + return &benchmarkHelperTrie{bloom: true, reqCount: 1} + }}, + "bloom16": {name: "bloom trie (16)", newInstance: func() requestBenchmark { + return &benchmarkHelperTrie{bloom: true, reqCount: 16} + }}, + "txsend": {name: "send transaction", newInstance: func() requestBenchmark { + return &benchmarkTxSend{} + }, outSizeCorr: 50}, + "txstatus": {name: "get transaction status", newInstance: func() requestBenchmark { + return &benchmarkTxStatus{} + }, outSizeCorr: 50}, +} + +// reqBenchMap defines the calculation method for different request costs based on +// the benchmark results +var reqBenchMap = []struct { + code uint64 // message code + // id contains a list of benchmarks that correspond to the cost of a single request + // the cost estimate of a single request is based on the highest benchmark result from the list + id []string + // idMax contains a list of benchmarks that correspond to the cost of a request with maxCount elements + // if idMax is not specified then the cost of additional request elements is the same as the cost + // of the single request + idMax []string + maxCount uint64 +}{ + {GetBlockHeadersMsg, []string{"header1n", "header1h"}, []string{"header192n", "header192hr"}, 192}, + {GetBlockBodiesMsg, []string{"body"}, nil, 1}, + {GetReceiptsMsg, []string{"receipts"}, nil, 1}, + {GetCodeMsg, []string{"code"}, nil, 1}, + {GetProofsV1Msg, []string{"proof"}, nil, 1}, + {GetProofsV2Msg, []string{"proof"}, nil, 1}, + {GetHeaderProofsMsg, []string{"cht1"}, []string{"cht16"}, 16}, + {GetHelperTrieProofsMsg, []string{"cht1", "bloom1"}, []string{"cht16", "bloom16"}, 16}, + {SendTxMsg, []string{"txsend"}, nil, 1}, + {SendTxV2Msg, []string{"txsend"}, nil, 1}, + {GetTxStatusMsg, []string{"txstatus"}, nil, 1}, +} + +// benchmarkSetup stores measurement data for a single benchmark type +type benchmarkSetup struct { + req requestBenchmark + id, name string + totalCount int + totalTime, avgTime time.Duration + maxInSize, maxOutSize uint32 + err error +} + +// reqBenchmarkKey is the database key for storing measurement data +var reqBenchmarkKey = []byte("_requestBenchmarks__") + +const ( + passCount = 10 // number of passes in which all benchmark types are measured + firstCount = 50 // request count for each type in the first pass (adjusted in subsequent passes) + totalBenchmarkTime = time.Second * 20 // targeted total run time for the given number of passes + discardAge = 100000 // block age after which a stored benchmark entry is discarded + rerunAge = 10000 // if the newest entry is older than rerunAge then a new benchmark is started + rerunCount = 5 // if the number of stored entries is less than rerunCount then a new benchmark is started +) + +// benchmarkData is the database storage format of benchmark results for a single type +type benchmarkData struct { + BlockNumber, AvgTime uint64 + MaxInSize, MaxOutSize uint32 +} + +type benchmarkDataByTime []benchmarkData + +func (s benchmarkDataByTime) Len() int { return len(s) } +func (s benchmarkDataByTime) Less(i, j int) bool { return s[i].AvgTime < s[j].AvgTime } +func (s benchmarkDataByTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// dataToCost calculates request cost estimates used by the flow control system +func dataToCost(id string, data []benchmarkData, inSizeCostFactor, outSizeCostFactor float64) uint64 { + var ( + maxInSize, maxOutSize uint32 + avgTime uint64 + ) + for _, d := range data { + if d.MaxInSize > maxInSize { + maxInSize = d.MaxInSize + } + if d.MaxOutSize > maxOutSize { + maxOutSize = d.MaxOutSize + } + } + var cost uint64 + if len(data) > 0 { + sort.Sort(benchmarkDataByTime(data)) + skip := len(data) / 5 + for i := skip; i < len(data)-skip; i++ { + avgTime += data[i].AvgTime + } + avgTime /= uint64(len(data) - skip*2) + bt := benchmarkTypes[id] + maxOutSize += bt.outSizeCorr + if bt.avgTimeCorr != 0 { + avgTime = uint64(float64(avgTime) * bt.avgTimeCorr) + } + cost = avgTime * 2 + } + inSizeCost := uint64(float64(maxInSize) * inSizeCostFactor * 1.25) + outSizeCost := uint64(float64(maxOutSize) * outSizeCostFactor * 1.25) + if inSizeCost > cost { + cost = inSizeCost + } + if outSizeCost > cost { + cost = outSizeCost + } + return cost +} + +// benchmarkCosts checks the database for existing entries and initiates a benchmark +// cycle for all types if necessary. It returns the cost list to be announced for +// clients and the minimum buffer limit that can be assigned to each client. +func (pm *ProtocolManager) benchmarkCosts(threadCount int, inSizeCostFactor, outSizeCostFactor float64) (costList RequestCostList, minBufLimit uint64) { + blockNumber := pm.blockchain.CurrentHeader().Number.Uint64() + allData := make(map[string][]benchmarkData) + run := false + for id, _ := range benchmarkTypes { + var data []benchmarkData + if enc, err := pm.chainDb.Get(append(reqBenchmarkKey, []byte(id)...)); err == nil { + if rlp.DecodeBytes(enc, &data) != nil { + data = nil + } + } + for len(data) > 0 && data[0].BlockNumber+discardAge <= blockNumber { + data = data[1:] + } + if len(data) < rerunCount || data[len(data)-1].BlockNumber+rerunAge <= blockNumber { + run = true + } + allData[id] = data + } + + if run { + res := pm.runBenchmark() + for _, r := range res { + if r.err == nil { + data := append(allData[r.id], benchmarkData{BlockNumber: blockNumber, AvgTime: uint64(r.avgTime) * uint64(threadCount), MaxInSize: r.maxInSize, MaxOutSize: r.maxOutSize}) + allData[r.id] = data + if enc, err := rlp.EncodeToBytes(data); err == nil { + pm.chainDb.Put(append(reqBenchmarkKey, []byte(r.id)...), enc) + } + } + } + } + + // calculate upper cost estimates based on AvgTime and MaxSize + costs := make(map[string]uint64) + for id, data := range allData { + costs[id] = dataToCost(id, data, inSizeCostFactor, outSizeCostFactor) + } + var maxAllCosts uint64 + // create linear cost functions for actual request types using reqBenchMap + res := make(RequestCostList, len(reqBenchMap)) + for i, m := range reqBenchMap { + res[i].MsgCode = m.code + var cost uint64 + for _, id := range m.id { + if c, ok := costs[id]; ok { + if c > cost { + cost = c + } + } else { + panic(nil) + } + } + if m.idMax == nil { + res[i].BaseCost = 0 + res[i].ReqCost = cost + } else { + var maxCost uint64 + for _, id := range m.idMax { + if c, ok := costs[id]; ok { + if c > maxCost { + maxCost = c + } + } else { + panic(nil) + } + } + if maxCost < cost { + maxCost = cost + } + if maxCost > maxAllCosts { + maxAllCosts = maxCost + } + dc := (maxCost - cost) / (m.maxCount - 1) + if cost < dc { + dc = maxCost / m.maxCount + cost = dc + } + res[i].BaseCost = cost - dc + res[i].ReqCost = dc + } + } + return res, maxAllCosts * 2 +} + +// runBenchmark runs a benchmark cycle for all benchmark types in the specified +// number of passes +func (pm *ProtocolManager) runBenchmark() []*benchmarkSetup { + log.Info("running benchmark") + setup := make([]*benchmarkSetup, len(benchmarkTypes)) + i := 0 + for id, bt := range benchmarkTypes { + setup[i] = &benchmarkSetup{id: id, name: bt.name, req: bt.newInstance()} + i++ + } + targetTime := totalBenchmarkTime / time.Duration(len(benchmarkTypes)*passCount) + for i := 0; i < passCount; i++ { + todo := make([]*benchmarkSetup, len(benchmarkTypes)) + copy(todo, setup) + for len(todo) > 0 { + // select a random element + index := rand.Intn(len(todo)) + next := todo[index] + todo[index] = todo[len(todo)-1] + todo = todo[:len(todo)-1] + + if next.err == nil { + // calculate request count + count := firstCount + if next.totalTime > 0 { + count = int(uint64(next.totalCount) * uint64(targetTime) / uint64(next.totalTime)) + } + if err := pm.measure(next, count); err != nil { + next.err = err + } + } + } + log.Info("benchmark completed", "percent", (i+1)*100/passCount) + } + + for _, s := range setup { + if s.err == nil { + s.avgTime = s.totalTime / time.Duration(s.totalCount) + log.Debug("benchmark result", "name", s.name, "avgTime", s.avgTime, "reqCount", s.totalCount, "maxInSize", s.maxInSize, "maxOutSize", s.maxOutSize) + } else { + log.Warn("benchmark failed", "name", s.name, "error", s.err) + } + } + return setup +} + +// meteredPipe implements p2p.MsgReadWriter and remembers the largest single +// message size sent through the pipe +type meteredPipe struct { + rw p2p.MsgReadWriter + maxSize uint32 +} + +func (m *meteredPipe) ReadMsg() (p2p.Msg, error) { + return m.rw.ReadMsg() +} + +func (m *meteredPipe) WriteMsg(msg p2p.Msg) error { + if msg.Size > m.maxSize { + m.maxSize = msg.Size + } + return m.rw.WriteMsg(msg) +} + +// measure runs a benchmark for a single type in a single pass, with the given +// number of requests +func (pm *ProtocolManager) measure(setup *benchmarkSetup, count int) error { + clientPipe, serverPipe := p2p.MsgPipe() + clientMeteredPipe := &meteredPipe{rw: clientPipe} + serverMeteredPipe := &meteredPipe{rw: serverPipe} + var id enode.ID + rand.Read(id[:]) + clientPeer := pm.newPeer(lpv2, NetworkId, p2p.NewPeer(id, "client", nil), clientMeteredPipe) + serverPeer := pm.newPeer(lpv2, NetworkId, p2p.NewPeer(id, "server", nil), serverMeteredPipe) + serverPeer.sendQueue = newExecQueue(count) + serverPeer.announceType = announceTypeNone + serverPeer.fcCosts = make(requestCostTable) + c := &requestCosts{} + for code, _ := range requests { + serverPeer.fcCosts[code] = c + } + serverPeer.fcParams = flowcontrol.ServerParams{BufLimit: 1, MinRecharge: 1} + serverPeer.fcClient = flowcontrol.NewClientNode(pm.server.fcManager, serverPeer.fcParams) + + if err := setup.req.init(pm, count); err != nil { + return err + } + + errCh := make(chan error, 10) + start := mclock.Now() + + go func() { + for i := 0; i < count; i++ { + if err := setup.req.request(clientPeer, i); err != nil { + errCh <- err + return + } + } + }() + go func() { + for i := 0; i < count; i++ { + if err := pm.handleMsg(serverPeer); err != nil { + errCh <- err + return + } + } + }() + go func() { + for i := 0; i < count; i++ { + msg, err := clientPipe.ReadMsg() + if err != nil { + errCh <- err + return + } + var i interface{} + msg.Decode(&i) + } + // at this point we can be sure that the other two + // goroutines finished successfully too + close(errCh) + }() + select { + case err := <-errCh: + if err != nil { + return err + } + case <-pm.quitSync: + clientPipe.Close() + serverPipe.Close() + return fmt.Errorf("Benchmark cancelled") + } + + setup.totalTime += time.Duration(mclock.Now() - start) + setup.totalCount += count + setup.maxInSize = clientMeteredPipe.maxSize + setup.maxOutSize = serverMeteredPipe.maxSize + clientPipe.Close() + serverPipe.Close() + //serverPeer.fcClient.Remove(pm.server.fcManager) + return nil +} + +// requestCostStats is a statistics tool that compares the distribution of actual +// request serving costs during normal operation to the costs estimated by the benchmark +type requestCostStats struct { + costs requestCostTable + stats map[uint64][]uint64 +} + +// newCostStats creates a new requestCostStats +func newCostStats(table requestCostTable) *requestCostStats { + stats := make(map[uint64][]uint64) + for code, _ := range table { + stats[code] = make([]uint64, 10) + } + return &requestCostStats{ + costs: table, + stats: stats, + } +} + +// update adds a new data point to the statistics +func (s *requestCostStats) update(msgCode, reqCnt, cost uint64) { + if s == nil { + return // not initialized yet during benchmark + } + c := s.costs[msgCode] + est := c.baseCost + reqCnt*c.reqCost + cost <<= 4 + l := 0 + for l < 9 && cost > est { + l++ + cost >>= 1 + } + ptr := &s.stats[msgCode][l] + atomic.AddUint64(ptr, 1) +} + +// printStats prints the distribution of real request cost relative to the estimates +func (s *requestCostStats) printStats() { + if s.stats == nil { + return + } + for code, arr := range s.stats { + log.Info("cost stats", "code", code, "1/16", arr[0], "1/8", arr[1], "1/4", arr[2], "1/2", arr[3], "1", arr[4], "2", arr[5], "4", arr[6], "8", arr[7], "16", arr[8], ">16", arr[9]) + } +} diff --git a/les/distributor.go b/les/distributor.go index f90765b624..3a429909ff 100644 --- a/les/distributor.go +++ b/les/distributor.go @@ -22,12 +22,15 @@ import ( "container/list" "sync" "time" + + "github.com/ethereum/go-ethereum/common/mclock" ) // requestDistributor implements a mechanism that distributes requests to // suitable peers, obeying flow control rules and prioritizing them in creation // order (even when a resend is necessary). type requestDistributor struct { + clock mclock.Clock reqQueue *list.List lastReqOrder uint64 peers map[distPeer]struct{} @@ -67,8 +70,9 @@ type distReq struct { } // newRequestDistributor creates a new request distributor -func newRequestDistributor(peers *peerSet, stopChn chan struct{}) *requestDistributor { +func newRequestDistributor(peers *peerSet, stopChn chan struct{}, clock mclock.Clock) *requestDistributor { d := &requestDistributor{ + clock: clock, reqQueue: list.New(), loopChn: make(chan struct{}, 2), stopChn: stopChn, @@ -148,7 +152,7 @@ func (d *requestDistributor) loop() { wait = distMaxWait } go func() { - time.Sleep(wait) + d.clock.Sleep(wait) d.loopChn <- struct{}{} }() break loop diff --git a/les/distributor_test.go b/les/distributor_test.go index 8c7621f26b..4990635267 100644 --- a/les/distributor_test.go +++ b/les/distributor_test.go @@ -23,6 +23,8 @@ import ( "sync" "testing" "time" + + "github.com/ethereum/go-ethereum/common/mclock" ) type testDistReq struct { @@ -121,7 +123,7 @@ func testRequestDistributor(t *testing.T, resend bool) { stop := make(chan struct{}) defer close(stop) - dist := newRequestDistributor(nil, stop) + dist := newRequestDistributor(nil, stop, &mclock.System{}) var peers [testDistPeerCount]*testDistPeer for i := range peers { peers[i] = &testDistPeer{} diff --git a/les/fetcher.go b/les/fetcher.go index aa3101af70..71a271fef1 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -559,7 +559,7 @@ func (f *lightFetcher) newFetcherDistReq(bestHash common.Hash, reqID uint64, bes f.lock.Unlock() cost := p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount)) - p.fcServer.QueueRequest(reqID, cost) + p.fcServer.QueuedRequest(reqID, cost) f.reqMu.Lock() f.requested[reqID] = fetchRequest{hash: bestHash, amount: bestAmount, peer: p, sent: mclock.Now()} f.reqMu.Unlock() diff --git a/les/flowcontrol/control.go b/les/flowcontrol/control.go index 3eec162600..f7a8e81b92 100644 --- a/les/flowcontrol/control.go +++ b/les/flowcontrol/control.go @@ -18,182 +18,326 @@ package flowcontrol import ( + "fmt" "sync" "time" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/log" ) -const fcTimeConst = time.Millisecond +const ( + // fcTimeConst is the time constant applied for MinRecharge during linear + // buffer recharge period + fcTimeConst = time.Millisecond + // DecParamDelay is applied at server side when decreasing bandwidth in order to + // avoid a buffer underrun error due to requests sent by the client before + // receiving the bandwidth update announcement + DecParamDelay = time.Second * 2 + // keepLogs is the duration of keeping logs; logging is not used if zero + keepLogs = 0 +) +// ServerParams are the flow control parameters specified by a server for a client +// +// Note: a server can assign different amounts of bandwidth to each client by giving +// different parameters to them. type ServerParams struct { BufLimit, MinRecharge uint64 } -type ClientNode struct { - params ServerParams - bufValue uint64 - lastTime mclock.AbsTime - lock sync.Mutex - cm *ClientManager - cmNode *cmNode +type scheduledUpdate struct { + time mclock.AbsTime + params ServerParams } +// ClientNode is the flow control system's representation of a client +// (used in server mode only) +type ClientNode struct { + params ServerParams + bufValue uint64 + lastTime mclock.AbsTime + updateSchedule []scheduledUpdate + sumCost uint64 // sum of req costs received from this client + accepted map[uint64]uint64 // value = sumCost after accepting the given req + lock sync.Mutex + cm *ClientManager + log *logger + cmNodeFields +} + +// NewClientNode returns a new ClientNode func NewClientNode(cm *ClientManager, params ServerParams) *ClientNode { node := &ClientNode{ cm: cm, params: params, bufValue: params.BufLimit, - lastTime: mclock.Now(), + lastTime: cm.clock.Now(), + accepted: make(map[uint64]uint64), } - node.cmNode = cm.addNode(node) + if keepLogs > 0 { + node.log = newLogger(keepLogs) + } + cm.init(node) return node } -func (peer *ClientNode) Remove(cm *ClientManager) { - cm.removeNode(peer.cmNode) +func (node *ClientNode) update(now mclock.AbsTime) { + for len(node.updateSchedule) > 0 && node.updateSchedule[0].time <= now { + node.recalcBV(node.updateSchedule[0].time) + node.updateParams(node.updateSchedule[0].params, now) + node.updateSchedule = node.updateSchedule[1:] + } + node.recalcBV(now) } -func (peer *ClientNode) recalcBV(time mclock.AbsTime) { - dt := uint64(time - peer.lastTime) - if time < peer.lastTime { +func (node *ClientNode) recalcBV(now mclock.AbsTime) { + dt := uint64(now - node.lastTime) + if now < node.lastTime { dt = 0 } - peer.bufValue += peer.params.MinRecharge * dt / uint64(fcTimeConst) - if peer.bufValue > peer.params.BufLimit { - peer.bufValue = peer.params.BufLimit + node.bufValue += node.params.MinRecharge * dt / uint64(fcTimeConst) + if node.bufValue > node.params.BufLimit { + node.bufValue = node.params.BufLimit } - peer.lastTime = time + if node.log != nil { + node.log.add(now, fmt.Sprintf("updated bv=%d MRR=%d BufLimit=%d", node.bufValue, node.params.MinRecharge, node.params.BufLimit)) + } + node.lastTime = now } -func (peer *ClientNode) AcceptRequest() (uint64, bool) { - peer.lock.Lock() - defer peer.lock.Unlock() +func (node *ClientNode) UpdateParams(params ServerParams) { + node.lock.Lock() + defer node.lock.Unlock() - time := mclock.Now() - peer.recalcBV(time) - return peer.bufValue, peer.cm.accept(peer.cmNode, time) -} - -func (peer *ClientNode) RequestProcessed(cost uint64) (bv, realCost uint64) { - peer.lock.Lock() - defer peer.lock.Unlock() - - time := mclock.Now() - peer.recalcBV(time) - peer.bufValue -= cost - rcValue, rcost := peer.cm.processed(peer.cmNode, time) - if rcValue < peer.params.BufLimit { - bv := peer.params.BufLimit - rcValue - if bv > peer.bufValue { - peer.bufValue = bv + now := node.cm.clock.Now() + node.update(now) + if params.MinRecharge >= node.params.MinRecharge { + node.updateSchedule = nil + node.updateParams(params, now) + } else { + for i, s := range node.updateSchedule { + if params.MinRecharge >= s.params.MinRecharge { + s.params = params + node.updateSchedule = node.updateSchedule[:i+1] + return + } } + node.updateSchedule = append(node.updateSchedule, scheduledUpdate{time: now + mclock.AbsTime(DecParamDelay), params: params}) } - return peer.bufValue, rcost } +func (node *ClientNode) updateParams(params ServerParams, now mclock.AbsTime) { + diff := params.BufLimit - node.params.BufLimit + if int64(diff) > 0 { + node.bufValue += diff + } else if node.bufValue > params.BufLimit { + node.bufValue = params.BufLimit + } + node.cm.updateParams(node, params, now) +} + +// AcceptRequest returns whether a new request can be accepted and the missing +// buffer amount if it was rejected due to a buffer underrun. If accepted, maxCost +// is deducted from the flow control buffer. +func (node *ClientNode) AcceptRequest(reqID, index, maxCost uint64) (accepted bool, bufShort uint64, priority int64) { + node.lock.Lock() + defer node.lock.Unlock() + + now := node.cm.clock.Now() + node.update(now) + if maxCost > node.bufValue { + if node.log != nil { + node.log.add(now, fmt.Sprintf("rejected reqID=%d bv=%d maxCost=%d", reqID, node.bufValue, maxCost)) + node.log.dump(now) + } + return false, maxCost - node.bufValue, 0 + } + node.bufValue -= maxCost + node.sumCost += maxCost + if node.log != nil { + node.log.add(now, fmt.Sprintf("accepted reqID=%d bv=%d maxCost=%d sumCost=%d", reqID, node.bufValue, maxCost, node.sumCost)) + } + node.accepted[index] = node.sumCost + return true, 0, node.cm.accepted(node, maxCost, now) +} + +// RequestProcessed should be called when the request has been processed +func (node *ClientNode) RequestProcessed(reqID, index, maxCost, realCost uint64) (bv uint64) { + node.lock.Lock() + defer node.lock.Unlock() + + now := node.cm.clock.Now() + node.update(now) + node.cm.processed(node, maxCost, realCost, now) + bv = node.bufValue + node.sumCost - node.accepted[index] + if node.log != nil { + node.log.add(now, fmt.Sprintf("processed reqID=%d bv=%d maxCost=%d realCost=%d sumCost=%d oldSumCost=%d reportedBV=%d", reqID, node.bufValue, maxCost, realCost, node.sumCost, node.accepted[index], bv)) + } + delete(node.accepted, index) + return +} + +// ServerNode is the flow control system's representation of a server +// (used in client mode only) type ServerNode struct { + clock mclock.Clock bufEstimate uint64 + bufRecharge bool lastTime mclock.AbsTime params ServerParams sumCost uint64 // sum of req costs sent to this server pending map[uint64]uint64 // value = sumCost after sending the given req + log *logger lock sync.RWMutex } -func NewServerNode(params ServerParams) *ServerNode { - return &ServerNode{ +// NewServerNode returns a new ServerNode +func NewServerNode(params ServerParams, clock mclock.Clock) *ServerNode { + node := &ServerNode{ + clock: clock, bufEstimate: params.BufLimit, - lastTime: mclock.Now(), + bufRecharge: false, + lastTime: clock.Now(), params: params, pending: make(map[uint64]uint64), } + if keepLogs > 0 { + node.log = newLogger(keepLogs) + } + return node } // UpdateParams updates flow control parameters -func (peer *ServerNode) UpdateParams(params ServerParams) { - peer.lock.Lock() - defer peer.lock.Unlock() +func (node *ServerNode) UpdateParams(params ServerParams) { + node.lock.Lock() + defer node.lock.Unlock() - peer.recalcBLE(mclock.Now()) - if params.BufLimit > peer.params.BufLimit { - peer.bufEstimate += params.BufLimit - peer.params.BufLimit + node.recalcBLE(mclock.Now()) + if params.BufLimit > node.params.BufLimit { + node.bufEstimate += params.BufLimit - node.params.BufLimit } else { - if peer.bufEstimate > params.BufLimit { - peer.bufEstimate = params.BufLimit + if node.bufEstimate > params.BufLimit { + node.bufEstimate = params.BufLimit } } - peer.params = params + node.params = params } -func (peer *ServerNode) recalcBLE(time mclock.AbsTime) { - dt := uint64(time - peer.lastTime) - if time < peer.lastTime { - dt = 0 +func (node *ServerNode) recalcBLE(now mclock.AbsTime) { + if now < node.lastTime { + return } - peer.bufEstimate += peer.params.MinRecharge * dt / uint64(fcTimeConst) - if peer.bufEstimate > peer.params.BufLimit { - peer.bufEstimate = peer.params.BufLimit + if node.bufRecharge { + dt := uint64(now - node.lastTime) + node.bufEstimate += node.params.MinRecharge * dt / uint64(fcTimeConst) + if node.bufEstimate >= node.params.BufLimit { + node.bufEstimate = node.params.BufLimit + node.bufRecharge = false + } + } + node.lastTime = now + if node.log != nil { + node.log.add(now, fmt.Sprintf("updated bufEst=%d MRR=%d BufLimit=%d", node.bufEstimate, node.params.MinRecharge, node.params.BufLimit)) } - peer.lastTime = time } // safetyMargin is added to the flow control waiting time when estimated buffer value is low const safetyMargin = time.Millisecond -func (peer *ServerNode) canSend(maxCost uint64) (time.Duration, float64) { - peer.recalcBLE(mclock.Now()) - maxCost += uint64(safetyMargin) * peer.params.MinRecharge / uint64(fcTimeConst) - if maxCost > peer.params.BufLimit { - maxCost = peer.params.BufLimit - } - if peer.bufEstimate >= maxCost { - return 0, float64(peer.bufEstimate-maxCost) / float64(peer.params.BufLimit) - } - return time.Duration((maxCost - peer.bufEstimate) * uint64(fcTimeConst) / peer.params.MinRecharge), 0 -} - // CanSend returns the minimum waiting time required before sending a request // with the given maximum estimated cost. Second return value is the relative // estimated buffer level after sending the request (divided by BufLimit). -func (peer *ServerNode) CanSend(maxCost uint64) (time.Duration, float64) { - peer.lock.RLock() - defer peer.lock.RUnlock() +func (node *ServerNode) CanSend(maxCost uint64) (time.Duration, float64) { + node.lock.RLock() + defer node.lock.RUnlock() - return peer.canSend(maxCost) -} - -// QueueRequest should be called when the request has been assigned to the given -// server node, before putting it in the send queue. It is mandatory that requests -// are sent in the same order as the QueueRequest calls are made. -func (peer *ServerNode) QueueRequest(reqID, maxCost uint64) { - peer.lock.Lock() - defer peer.lock.Unlock() - - peer.bufEstimate -= maxCost - peer.sumCost += maxCost - peer.pending[reqID] = peer.sumCost -} - -// GotReply adjusts estimated buffer value according to the value included in -// the latest request reply. -func (peer *ServerNode) GotReply(reqID, bv uint64) { - - peer.lock.Lock() - defer peer.lock.Unlock() - - if bv > peer.params.BufLimit { - bv = peer.params.BufLimit + now := node.clock.Now() + node.recalcBLE(now) + maxCost += uint64(safetyMargin) * node.params.MinRecharge / uint64(fcTimeConst) + if maxCost > node.params.BufLimit { + maxCost = node.params.BufLimit } - sc, ok := peer.pending[reqID] + if node.bufEstimate >= maxCost { + relBuf := float64(node.bufEstimate-maxCost) / float64(node.params.BufLimit) + if node.log != nil { + node.log.add(now, fmt.Sprintf("canSend bufEst=%d maxCost=%d true relBuf=%f", node.bufEstimate, maxCost, relBuf)) + } + return 0, relBuf + } + timeLeft := time.Duration((maxCost - node.bufEstimate) * uint64(fcTimeConst) / node.params.MinRecharge) + if node.log != nil { + node.log.add(now, fmt.Sprintf("canSend bufEst=%d maxCost=%d false timeLeft=%v", node.bufEstimate, maxCost, timeLeft)) + } + return timeLeft, 0 +} + +// QueuedRequest should be called when the request has been assigned to the given +// server node, before putting it in the send queue. It is mandatory that requests +// are sent in the same order as the QueuedRequest calls are made. +func (node *ServerNode) QueuedRequest(reqID, maxCost uint64) { + node.lock.Lock() + defer node.lock.Unlock() + + now := node.clock.Now() + node.recalcBLE(now) + // Note: we do not know when requests actually arrive to the server so bufRecharge + // is not turned on here if buffer was full; in this case it is going to be turned + // on by the first reply's bufValue feedback + if node.bufEstimate >= maxCost { + node.bufEstimate -= maxCost + } else { + log.Error("Queued request with insufficient buffer estimate") + node.bufEstimate = 0 + } + node.sumCost += maxCost + node.pending[reqID] = node.sumCost + if node.log != nil { + node.log.add(now, fmt.Sprintf("queued reqID=%d bufEst=%d maxCost=%d sumCost=%d", reqID, node.bufEstimate, maxCost, node.sumCost)) + } +} + +// ReceivedReply adjusts estimated buffer value according to the value included in +// the latest request reply. +func (node *ServerNode) ReceivedReply(reqID, bv uint64) { + node.lock.Lock() + defer node.lock.Unlock() + + now := node.clock.Now() + node.recalcBLE(now) + if bv > node.params.BufLimit { + bv = node.params.BufLimit + } + sc, ok := node.pending[reqID] if !ok { return } - delete(peer.pending, reqID) - cc := peer.sumCost - sc - peer.bufEstimate = 0 + delete(node.pending, reqID) + cc := node.sumCost - sc + newEstimate := uint64(0) if bv > cc { - peer.bufEstimate = bv - cc + newEstimate = bv - cc + } + if newEstimate > node.bufEstimate { + // Note: we never reduce the buffer estimate based on the reported value because + // this can only happen because of the delayed delivery of the latest reply. + // The lowest estimate based on the previous reply can still be considered valid. + node.bufEstimate = newEstimate + } + + node.bufRecharge = node.bufEstimate < node.params.BufLimit + node.lastTime = now + if node.log != nil { + node.log.add(now, fmt.Sprintf("received reqID=%d bufEst=%d reportedBv=%d sumCost=%d oldSumCost=%d", reqID, node.bufEstimate, bv, node.sumCost, sc)) + } +} + +// DumpLogs dumps the event log if logging is used +func (node *ServerNode) DumpLogs() { + node.lock.Lock() + defer node.lock.Unlock() + + if node.log != nil { + node.log.dump(node.clock.Now()) } - peer.lastTime = mclock.Now() } diff --git a/les/flowcontrol/logger.go b/les/flowcontrol/logger.go new file mode 100644 index 0000000000..3c7e142892 --- /dev/null +++ b/les/flowcontrol/logger.go @@ -0,0 +1,66 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package flowcontrol implements a client side flow control mechanism +package flowcontrol + +import ( + "fmt" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +// logger collects events in string format and discards events older than the +// "keep" parameter +type logger struct { + events map[uint64]logEvent + writePtr, delPtr uint64 + keep time.Duration +} + +// logEvent describes a single event +type logEvent struct { + time mclock.AbsTime + event string +} + +// newLogger creates a new logger +func newLogger(keep time.Duration) *logger { + return &logger{ + events: make(map[uint64]logEvent), + keep: keep, + } +} + +// add adds a new event and discards old events if possible +func (l *logger) add(now mclock.AbsTime, event string) { + keepAfter := now - mclock.AbsTime(l.keep) + for l.delPtr < l.writePtr && l.events[l.delPtr].time <= keepAfter { + delete(l.events, l.delPtr) + l.delPtr++ + } + l.events[l.writePtr] = logEvent{now, event} + l.writePtr++ +} + +// dump prints all stored events +func (l *logger) dump(now mclock.AbsTime) { + for i := l.delPtr; i < l.writePtr; i++ { + e := l.events[i] + fmt.Println(time.Duration(e.time-now), e.event) + } +} diff --git a/les/flowcontrol/manager.go b/les/flowcontrol/manager.go index 28cc6f0fe7..cb87dc75d0 100644 --- a/les/flowcontrol/manager.go +++ b/les/flowcontrol/manager.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -18,207 +18,269 @@ package flowcontrol import ( + "fmt" "sync" - "time" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/common/prque" ) -const rcConst = 1000000 - -type cmNode struct { - node *ClientNode - lastUpdate mclock.AbsTime - serving, recharging bool - rcWeight uint64 - rcValue, rcDelta, startValue int64 - finishRecharge mclock.AbsTime +// cmNodeFields are ClientNode fields used by the client manager +// Note: these fields are locked by the client manager's mutex +type cmNodeFields struct { + corrBufValue int64 // buffer value adjusted with the extra recharge amount + rcLastIntValue int64 // past recharge integrator value when corrBufValue was last updated + rcFullIntValue int64 // future recharge integrator value when corrBufValue will reach maximum + queueIndex int // position in the recharge queue (-1 if not queued) } -func (node *cmNode) update(time mclock.AbsTime) { - dt := int64(time - node.lastUpdate) - node.rcValue += node.rcDelta * dt / rcConst - node.lastUpdate = time - if node.recharging && time >= node.finishRecharge { - node.recharging = false - node.rcDelta = 0 - node.rcValue = 0 - } -} - -func (node *cmNode) set(serving bool, simReqCnt, sumWeight uint64) { - if node.serving && !serving { - node.recharging = true - sumWeight += node.rcWeight - } - node.serving = serving - if node.recharging && serving { - node.recharging = false - sumWeight -= node.rcWeight - } - - node.rcDelta = 0 - if serving { - node.rcDelta = int64(rcConst / simReqCnt) - } - if node.recharging { - node.rcDelta = -int64(node.node.cm.rcRecharge * node.rcWeight / sumWeight) - node.finishRecharge = node.lastUpdate + mclock.AbsTime(node.rcValue*rcConst/(-node.rcDelta)) - } -} +// FixedPointMultiplier is applied to the recharge integrator and the recharge curve. +// +// Note: fixed point arithmetic is required for the integrator because it is a +// constantly increasing value that can wrap around int64 limits (which behavior is +// also supported by the priority queue). A floating point value would gradually lose +// precision in this application. +// The recharge curve and all recharge values are encoded as fixed point because +// sumRecharge is frequently updated by adding or subtracting individual recharge +// values and perfect precision is required. +const FixedPointMultiplier = 1000000 +// ClientManager controls the bandwidth assigned to the clients of a server. +// Since ServerParams guarantee a safe lower estimate for processable requests +// even in case of all clients being active, ClientManager calculates a +// corrigated buffer value and usually allows a higher remaining buffer value +// to be returned with each reply. type ClientManager struct { - lock sync.Mutex - nodes map[*cmNode]struct{} - simReqCnt, sumWeight, rcSumValue uint64 - maxSimReq, maxRcSum uint64 - rcRecharge uint64 - resumeQueue chan chan bool - time mclock.AbsTime + clock mclock.Clock + lock sync.Mutex + nodes map[*ClientNode]struct{} + enabledCh chan struct{} + + curve PieceWiseLinear + sumRecharge uint64 + // recharge integrator is increasing in each moment with a rate of + // (totalRecharge / sumRecharge)*FixedPointMultiplier or 0 if sumRecharge==0 + rcLastUpdate mclock.AbsTime // last time the recharge integrator was updated + rcLastIntValue int64 // last updated value of the recharge integrator + // recharge queue is a priority queue with currently recharging client nodes + // as elements. The priority value is rcFullIntValue which allows to quickly + // determine which client will first finish recharge. + rcQueue *prque.Prque } -func NewClientManager(rcTarget, maxSimReq, maxRcSum uint64) *ClientManager { +// NewClientManager returns a new client manager. +// Client manager enhances flow control performance by allowing client buffers +// to recharge quicker than the minimum guaranteed recharge rate if possible. +// The sum of all minimum recharge rates (sumRecharge) is updated each time +// a clients starts or finishes buffer recharging. Then an adjusted total +// recharge rate is calculated using a piecewise linear recharge curve: +// +// totalRecharge = curve(sumRecharge) +// (totalRecharge >= sumRecharge is enforced) +// +// Then the "bonus" buffer recharge is distributed between currently recharging +// clients proportionally to their minimum recharge rates. +// +// Note: total recharge is proportional to the average number of parallel running +// serving threads. A recharge value of 1000000 corresponds to one thread in average. +// The maximum number of allowed serving threads should always be considerably +// higher than the targeted average number. +// +// Note 2: although it is possible to specify a curve allowing the total target +// recharge starting from zero sumRecharge, it makes sense to add a linear ramp +// starting from zero in order to not let a single low-priority client use up +// the entire server capacity and thus ensure quick availability for others at +// any moment. +func NewClientManager(curve PieceWiseLinear, clock mclock.Clock) *ClientManager { cm := &ClientManager{ - nodes: make(map[*cmNode]struct{}), - resumeQueue: make(chan chan bool), - rcRecharge: rcConst * rcConst / (100*rcConst/rcTarget - rcConst), - maxSimReq: maxSimReq, - maxRcSum: maxRcSum, + clock: clock, + nodes: make(map[*ClientNode]struct{}), + rcQueue: prque.New(func(a interface{}, i int) { a.(*ClientNode).queueIndex = i }), + curve: curve, } - go cm.queueProc() return cm } -func (self *ClientManager) Stop() { - self.lock.Lock() - defer self.lock.Unlock() +// SetRechargeCurve updates the recharge curve +func (cm *ClientManager) SetRechargeCurve(curve PieceWiseLinear) { + cm.lock.Lock() + defer cm.lock.Unlock() - // signal any waiting accept routines to return false - self.nodes = make(map[*cmNode]struct{}) - close(self.resumeQueue) + cm.updateRecharge(cm.clock.Now()) + cm.curve = curve } -func (self *ClientManager) addNode(cnode *ClientNode) *cmNode { - time := mclock.Now() - node := &cmNode{ - node: cnode, - lastUpdate: time, - finishRecharge: time, - rcWeight: 1, +// init initializes the ClientManager specific fields of a ClientNode structure +func (cm *ClientManager) init(node *ClientNode) { + cm.lock.Lock() + defer cm.lock.Unlock() + + node.corrBufValue = int64(node.params.BufLimit) + node.rcLastIntValue = cm.rcLastIntValue + node.queueIndex = -1 +} + +// accepted deduces the upper estimate for request cost from the buffer and returns a priority +// value based on current buffer status which is used by the serving queue. +func (cm *ClientManager) accepted(node *ClientNode, maxCost uint64, now mclock.AbsTime) (priority int64) { + cm.lock.Lock() + defer cm.lock.Unlock() + + cm.updateNodeRc(node, -int64(maxCost), &node.params, now) + rcTime := (node.params.BufLimit - uint64(node.corrBufValue)) * FixedPointMultiplier / node.params.MinRecharge + return -int64(now) - int64(rcTime) +} + +// processed updates the client buffer according to actual request cost after +// serving has been finished. +// +// Note: processed should always be called for all accepted requests +func (cm *ClientManager) processed(node *ClientNode, maxCost, realCost uint64, now mclock.AbsTime) { + cm.lock.Lock() + defer cm.lock.Unlock() + + if realCost > maxCost { + realCost = maxCost } - self.lock.Lock() - defer self.lock.Unlock() - - self.nodes[node] = struct{}{} - self.update(mclock.Now()) - return node -} - -func (self *ClientManager) removeNode(node *cmNode) { - self.lock.Lock() - defer self.lock.Unlock() - - time := mclock.Now() - self.stop(node, time) - delete(self.nodes, node) - self.update(time) -} - -// recalc sumWeight -func (self *ClientManager) updateNodes(time mclock.AbsTime) (rce bool) { - var sumWeight, rcSum uint64 - for node := range self.nodes { - rc := node.recharging - node.update(time) - if rc && !node.recharging { - rce = true + cm.updateNodeRc(node, int64(maxCost-realCost), &node.params, now) + if uint64(node.corrBufValue) > node.bufValue { + if node.log != nil { + node.log.add(now, fmt.Sprintf("corrected bv=%d oldBv=%d", node.corrBufValue, node.bufValue)) } - if node.recharging { - sumWeight += node.rcWeight - } - rcSum += uint64(node.rcValue) + node.bufValue = uint64(node.corrBufValue) } - self.sumWeight = sumWeight - self.rcSumValue = rcSum - return } -func (self *ClientManager) update(time mclock.AbsTime) { - for { - firstTime := time - for node := range self.nodes { - if node.recharging && node.finishRecharge < firstTime { - firstTime = node.finishRecharge - } +func (cm *ClientManager) updateParams(node *ClientNode, params ServerParams, now mclock.AbsTime) { + cm.lock.Lock() + defer cm.lock.Unlock() + + cm.updateNodeRc(node, 0, ¶ms, now) +} + +// updateRecharge updates the recharge integrator and checks the recharge queue +// for nodes with recently filled buffers +func (cm *ClientManager) updateRecharge(now mclock.AbsTime) { + lastUpdate := cm.rcLastUpdate + cm.rcLastUpdate = now + // updating is done in multiple steps if node buffers are filled and sumRecharge + // is decreased before the given target time + for cm.sumRecharge > 0 { + bonusRatio := cm.curve.ValueAt(cm.sumRecharge) / float64(cm.sumRecharge) + if bonusRatio < 1 { + bonusRatio = 1 } - if self.updateNodes(firstTime) { - for node := range self.nodes { - if node.recharging { - node.set(node.serving, self.simReqCnt, self.sumWeight) - } - } - } else { - self.time = time + dt := now - lastUpdate + // fetch the client that finishes first + + if cm.rcQueue.Empty() { // debug + fmt.Println("cm.sumRecharge", cm.sumRecharge) + panic("rcQueue is empty") + } + + rcqNode := cm.rcQueue.PopItem().(*ClientNode) // if sumRecharge > 0 then the queue cannot be empty + // check whether it has already finished + dtNext := mclock.AbsTime(float64(rcqNode.rcFullIntValue-cm.rcLastIntValue) / bonusRatio) + if dt < dtNext { + // not finished yet, put it back, update integrator according + // to current bonusRatio and return + cm.rcQueue.Push(rcqNode, -rcqNode.rcFullIntValue) + cm.rcLastIntValue += int64(bonusRatio * float64(dt)) return } - } -} - -func (self *ClientManager) canStartReq() bool { - return self.simReqCnt < self.maxSimReq && self.rcSumValue < self.maxRcSum -} - -func (self *ClientManager) queueProc() { - for rc := range self.resumeQueue { - for { - time.Sleep(time.Millisecond * 10) - self.lock.Lock() - self.update(mclock.Now()) - cs := self.canStartReq() - self.lock.Unlock() - if cs { - break - } + // finished recharging, update corrBufValue and sumRecharge if necessary and do next step + if rcqNode.corrBufValue < int64(rcqNode.params.BufLimit) { + rcqNode.corrBufValue = int64(rcqNode.params.BufLimit) + cm.sumRecharge -= rcqNode.params.MinRecharge } - close(rc) + lastUpdate += dtNext + cm.rcLastIntValue = rcqNode.rcFullIntValue } } -func (self *ClientManager) accept(node *cmNode, time mclock.AbsTime) bool { - self.lock.Lock() - defer self.lock.Unlock() +// updateNodeRc updates a node's corrBufValue and adds an external correction value. +// It also adds or removes the rcQueue entry and updates ServerParams and sumRecharge if necessary. +func (cm *ClientManager) updateNodeRc(node *ClientNode, bvc int64, params *ServerParams, now mclock.AbsTime) { + cm.updateRecharge(now) + wasFull := true + if node.corrBufValue != int64(node.params.BufLimit) { + wasFull = false + node.corrBufValue += (cm.rcLastIntValue - node.rcLastIntValue) * int64(node.params.MinRecharge) / FixedPointMultiplier + if node.corrBufValue > int64(node.params.BufLimit) { + node.corrBufValue = int64(node.params.BufLimit) + } + node.rcLastIntValue = cm.rcLastIntValue + } + node.corrBufValue += bvc + if node.corrBufValue < 0 { + node.corrBufValue = 0 + } + diff := int64(params.BufLimit - node.params.BufLimit) + if diff > 0 { + node.corrBufValue += diff + } + isFull := false + if node.corrBufValue >= int64(params.BufLimit) { + node.corrBufValue = int64(params.BufLimit) + isFull = true + } + if !wasFull { + cm.sumRecharge -= node.params.MinRecharge + } + if params != &node.params { + node.params = *params + } + if !isFull { + cm.sumRecharge += node.params.MinRecharge + if node.queueIndex != -1 { + cm.rcQueue.Remove(node.queueIndex) + } + node.rcLastIntValue = cm.rcLastIntValue + node.rcFullIntValue = cm.rcLastIntValue + (int64(node.params.BufLimit)-node.corrBufValue)*FixedPointMultiplier/int64(node.params.MinRecharge) + cm.rcQueue.Push(node, -node.rcFullIntValue) + } +} - self.update(time) - if !self.canStartReq() { - resume := make(chan bool) - self.lock.Unlock() - self.resumeQueue <- resume - <-resume - self.lock.Lock() - if _, ok := self.nodes[node]; !ok { - return false // reject if node has been removed or manager has been stopped +// PieceWiseLinear is used to describe recharge curves +type PieceWiseLinear []struct{ X, Y uint64 } + +// ValueAt returns the curve's value at a given point +func (pwl PieceWiseLinear) ValueAt(x uint64) float64 { + l := 0 + h := len(pwl) + if h == 0 { + return 0 + } + for h != l { + m := (l + h) / 2 + if x > pwl[m].X { + l = m + 1 + } else { + h = m } } - self.simReqCnt++ - node.set(true, self.simReqCnt, self.sumWeight) - node.startValue = node.rcValue - self.update(self.time) + if l == 0 { + return float64(pwl[0].Y) + } + l-- + if h == len(pwl) { + return float64(pwl[l].Y) + } + dx := pwl[h].X - pwl[l].X + if dx < 1 { + return float64(pwl[l].Y) + } + return float64(pwl[l].Y) + float64(pwl[h].Y-pwl[l].Y)*float64(x-pwl[l].X)/float64(dx) +} + +// Valid returns true if the X coordinates of the curve points are non-strictly monotonic +func (pwl PieceWiseLinear) Valid() bool { + var lastX uint64 + for _, i := range pwl { + if i.X < lastX { + return false + } + lastX = i.X + } return true } - -func (self *ClientManager) stop(node *cmNode, time mclock.AbsTime) { - if node.serving { - self.update(time) - self.simReqCnt-- - node.set(false, self.simReqCnt, self.sumWeight) - self.update(time) - } -} - -func (self *ClientManager) processed(node *cmNode, time mclock.AbsTime) (rcValue, rcCost uint64) { - self.lock.Lock() - defer self.lock.Unlock() - - self.stop(node, time) - return uint64(node.rcValue), uint64(node.rcValue - node.startValue) -} diff --git a/les/flowcontrol/manager_test.go b/les/flowcontrol/manager_test.go new file mode 100644 index 0000000000..064e278b12 --- /dev/null +++ b/les/flowcontrol/manager_test.go @@ -0,0 +1,124 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package flowcontrol implements a client side flow control mechanism +package flowcontrol + +import ( + "math/rand" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +type testNode struct { + node *ClientNode + bufLimit, bandwidth uint64 + waitUntil mclock.AbsTime + index, totalCost uint64 +} + +const ( + testMaxCost = 1000000 + testLength = 100000 +) + +// testConstantTotalBandwidth simulates multiple request sender nodes and verifies +// whether the total amount of served requests matches the expected value based on +// the total bandwidth and the duration of the test. +// Some nodes are sending requests occasionally so that their buffer should regularly +// reach the maximum while other nodes (the "max capacity nodes") are sending at the +// maximum permitted rate. The max capacity nodes are changed multiple times during +// a single test. +func TestConstantTotalBandwidth(t *testing.T) { + testConstantTotalBandwidth(t, 10, 1, 0) + testConstantTotalBandwidth(t, 10, 1, 1) + testConstantTotalBandwidth(t, 30, 1, 0) + testConstantTotalBandwidth(t, 30, 2, 3) + testConstantTotalBandwidth(t, 100, 1, 0) + testConstantTotalBandwidth(t, 100, 3, 5) + testConstantTotalBandwidth(t, 100, 5, 10) +} + +func testConstantTotalBandwidth(t *testing.T, nodeCount, maxCapacityNodes, randomSend int) { + clock := &mclock.Simulated{} + nodes := make([]*testNode, nodeCount) + var totalBandwidth uint64 + for i, _ := range nodes { + nodes[i] = &testNode{bandwidth: uint64(50000 + rand.Intn(100000))} + totalBandwidth += nodes[i].bandwidth + } + m := NewClientManager(PieceWiseLinear{{0, totalBandwidth}}, clock) + for _, n := range nodes { + n.bufLimit = n.bandwidth * 6000 //uint64(2000+rand.Intn(10000)) + n.node = NewClientNode(m, ServerParams{BufLimit: n.bufLimit, MinRecharge: n.bandwidth}) + } + maxNodes := make([]int, maxCapacityNodes) + for i, _ := range maxNodes { + // we don't care if some indexes are selected multiple times + // in that case we have fewer max nodes + maxNodes[i] = rand.Intn(nodeCount) + } + + for i := 0; i < testLength; i++ { + now := clock.Now() + for _, idx := range maxNodes { + for nodes[idx].send(t, now) { + } + } + if rand.Intn(testLength) < maxCapacityNodes*3 { + maxNodes[rand.Intn(maxCapacityNodes)] = rand.Intn(nodeCount) + } + + sendCount := randomSend + for sendCount > 0 { + if nodes[rand.Intn(nodeCount)].send(t, now) { + sendCount-- + } + } + + clock.Run(time.Millisecond) + } + + var totalCost uint64 + for _, n := range nodes { + totalCost += n.totalCost + } + ratio := float64(totalCost) / float64(totalBandwidth) / testLength + if ratio < 0.98 || ratio > 1.02 { + t.Errorf("totalCost/totalBandwidth/testLength ratio incorrect (expected: 1, got: %f)", ratio) + } + +} + +func (n *testNode) send(t *testing.T, now mclock.AbsTime) bool { + if now < n.waitUntil { + return false + } + n.index++ + if ok, _, _ := n.node.AcceptRequest(0, n.index, testMaxCost); !ok { + t.Fatalf("Rejected request after expected waiting time has passed") + } + rcost := uint64(rand.Int63n(testMaxCost)) + bv := n.node.RequestProcessed(0, n.index, testMaxCost, rcost) + if bv < testMaxCost { + n.waitUntil = now + mclock.AbsTime((testMaxCost-bv)*1001000/n.bandwidth) + } + //n.waitUntil = now + mclock.AbsTime(float64(testMaxCost)*1001000/float64(n.bandwidth)*(1-float64(bv)/float64(n.bufLimit))) + n.totalCost += rcost + return true +} diff --git a/les/freeclient.go b/les/freeclient.go index 5ee607be8f..9fbc43443c 100644 --- a/les/freeclient.go +++ b/les/freeclient.go @@ -96,6 +96,11 @@ func (f *freeClientPool) connect(address string, disconnectFn func()) bool { if f.closed { return false } + + if f.connectedLimit == 0 { + log.Debug("Client rejected", "address", address) + return false + } e := f.addressMap[address] now := f.clock.Now() var recentUsage int64 @@ -115,12 +120,7 @@ func (f *freeClientPool) connect(address string, disconnectFn func()) bool { i := f.connPool.PopItem().(*freeClientPoolEntry) if e.linUsage+int64(connectedBias)-i.linUsage < 0 { // kick it out and accept the new client - f.connPool.Remove(i.index) - f.calcLogUsage(i, now) - i.connected = false - f.disconnPool.Push(i, -i.logUsage) - log.Debug("Client kicked out", "address", i.address) - i.disconnectFn() + f.dropClient(i, now) } else { // keep the old client and reject the new one f.connPool.Push(i, i.linUsage) @@ -163,6 +163,31 @@ func (f *freeClientPool) disconnect(address string) { log.Debug("Client disconnected", "address", address) } +// setConnLimit sets the maximum number of free client slots and also drops +// some peers if necessary +func (f *freeClientPool) setConnLimit(newLimit int) { + f.lock.Lock() + defer f.lock.Unlock() + + f.connectedLimit = newLimit + now := mclock.Now() + for f.connPool.Size() > f.connectedLimit { + i := f.connPool.PopItem().(*freeClientPoolEntry) + f.dropClient(i, now) + } +} + +// dropClient disconnects a client and also moves it from the connected to the +// disconnected pool +func (f *freeClientPool) dropClient(i *freeClientPoolEntry, now mclock.AbsTime) { + f.connPool.Remove(i.index) + f.calcLogUsage(i, now) + i.connected = false + f.disconnPool.Push(i, -i.logUsage) + log.Debug("Client kicked out", "address", i.address) + i.disconnectFn() +} + // logOffset calculates the time-dependent offset for the logarithmic // representation of recent usage func (f *freeClientPool) logOffset(now mclock.AbsTime) int64 { diff --git a/les/handler.go b/les/handler.go index 95491dc3f8..91d57546e2 100644 --- a/les/handler.go +++ b/les/handler.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" @@ -90,21 +91,25 @@ type txPool interface { } type ProtocolManager struct { - lightSync bool - txpool txPool - txrelay *LesTxRelay - networkId uint64 - chainConfig *params.ChainConfig - iConfig *light.IndexerConfig - blockchain BlockChain - chainDb ethdb.Database - odr *LesOdr - server *LesServer - serverPool *serverPool - clientPool *freeClientPool - lesTopic discv5.Topic - reqDist *requestDistributor - retriever *retrieveManager + lightSync bool + txpool txPool + txrelay *LesTxRelay + networkId uint64 + chainConfig *params.ChainConfig + iConfig *light.IndexerConfig + blockchain BlockChain + chainDb ethdb.Database + odr *LesOdr + server *LesServer + serverPool *serverPool + clientPool *freeClientPool + freeClientBw uint64 + vipClientPool *vipClientPool + lesTopic discv5.Topic + reqDist *requestDistributor + retriever *retrieveManager + servingQueue *servingQueue + inSizeCostFactor, outSizeCostFactor float64 downloader *downloader.Downloader fetcher *lightFetcher @@ -165,6 +170,8 @@ func NewProtocolManager( if odr != nil { manager.retriever = odr.retriever manager.reqDist = odr.retriever.dist + } else { + manager.servingQueue = newServingQueue(int64(time.Millisecond * 10)) } if ulcConfig != nil { @@ -181,7 +188,6 @@ func NewProtocolManager( manager.peers.notify((*downloaderPeerNotify)(manager)) manager.fetcher = newLightFetcher(manager) } - return manager, nil } @@ -190,13 +196,38 @@ func (pm *ProtocolManager) removePeer(id string) { pm.peers.Unregister(id) } +// maxFreePeers returns the maximum number of free client slots based on the number +// and total bandwidth of other clients +func (pm *ProtocolManager) maxFreePeers(otherPeers int, otherBw uint64) int { + if otherPeers >= pm.maxPeers || otherBw >= pm.server.totalBandwidth { + return 0 + } + maxPeers := int((pm.server.totalBandwidth - otherBw) / pm.freeClientBw) + if maxPeers <= pm.maxPeers-otherPeers { + return maxPeers + } + return pm.maxPeers - otherPeers +} + func (pm *ProtocolManager) Start(maxPeers int) { pm.maxPeers = maxPeers + if pm.server != nil && maxPeers > 0 { + pm.freeClientBw = pm.server.totalBandwidth / uint64(maxPeers) + if pm.freeClientBw < pm.server.minBandwidth { + pm.freeClientBw = pm.server.minBandwidth + } + if pm.freeClientBw > 0 { + pm.server.defParams = flowcontrol.ServerParams{ + BufLimit: pm.freeClientBw * bufLimitRatio, + MinRecharge: pm.freeClientBw, + } + } + } if pm.lightSync { go pm.syncer() } else { - pm.clientPool = newFreeClientPool(pm.chainDb, maxPeers, 10000, mclock.System{}) + pm.clientPool = newFreeClientPool(pm.chainDb, pm.maxFreePeers(0, 0), 10000, mclock.System{}) go func() { for range pm.newPeerCh { } @@ -287,18 +318,6 @@ func (pm *ProtocolManager) handle(p *peer) error { return err } - if !pm.lightSync && !p.Peer.Info().Network.Trusted { - addr, ok := p.RemoteAddr().(*net.TCPAddr) - // test peer address is not a tcp address, don't use client pool if can not typecast - if ok { - id := addr.IP.String() - if !pm.clientPool.connect(id, func() { go pm.removePeer(p.id) }) { - return p2p.DiscTooManyPeers - } - defer pm.clientPool.disconnect(id) - } - } - if rw, ok := p.rw.(*meteredMsgReadWriter); ok { rw.Init(p.version) } @@ -309,12 +328,89 @@ func (pm *ProtocolManager) handle(p *peer) error { return err } defer func() { - if pm.server != nil && pm.server.fcManager != nil && p.fcClient != nil { - p.fcClient.Remove(pm.server.fcManager) - } pm.removePeer(p.id) }() + if !pm.lightSync && !p.Peer.Info().Network.Trusted { + var freeId string + if addr, ok := p.RemoteAddr().(*net.TCPAddr); ok { + freeId = addr.IP.String() + } + + var ( + free, vip bool + lock sync.Mutex // lock protects access to the free and vip flags + ) + + defer func() { + lock.Lock() + if free { + pm.clientPool.disconnect(freeId) + } + lock.Unlock() + }() + + updateBw := func(bw uint64) { + lock.Lock() + defer lock.Unlock() + + if !vip && bw != 0 { + // switch to vip mode + if free { + pm.clientPool.disconnect(freeId) + free = false + } + vip = true + p.updateBandwidth(bw) + } + if vip { + if bw == 0 { + // priority revoked; switch to free client mode or drop + if freeId != "" { + if !pm.clientPool.connect(freeId, func() { go pm.removePeer(p.id) }) { + pm.removePeer(p.id) + return + } + free = true + } + vip = false + p.updateBandwidth(pm.freeClientBw) + } else { + // just update vip bandwidth + p.updateBandwidth(bw) + } + } + } + + lock.Lock() + if pm.vipClientPool != nil { + // the vip client pool registers currently connected non-vip clients too + // in order to be able to notify them if they get priority while connected + vipBw, ok := pm.vipClientPool.connect(p.ID(), updateBw) + if !ok { + lock.Unlock() + return p2p.DiscAlreadyConnected + } + // always unregister + defer pm.vipClientPool.disconnect(p.ID()) + if vipBw != 0 { + vip = true + p.updateBandwidth(vipBw) + } + } + + if !vip && freeId != "" { + // if freeId == "" then we are in test mode and let the client connect + // without entering the free client pool + if !pm.clientPool.connect(freeId, func() { go pm.removePeer(p.id) }) { + lock.Unlock() + return p2p.DiscTooManyPeers + } + free = true + } + lock.Unlock() + } + // Register the peer in the downloader. If the downloader considers it banned, we disconnect if pm.lightSync { p.lock.Lock() @@ -329,31 +425,18 @@ func (pm *ProtocolManager) handle(p *peer) error { } } - stop := make(chan struct{}) - defer close(stop) - go func() { - // new block announce loop - for { - select { - case announce := <-p.announceChn: - p.SendAnnounce(announce) - case <-stop: - return - } - } - }() - // main loop. handle incoming messages. for { if err := pm.handleMsg(p); err != nil { p.Log().Debug("Light Ethereum message handling failed", "err", err) + if p.fcServer != nil { + p.fcServer.DumpLogs() + } return err } } } -var reqList = []uint64{GetBlockHeadersMsg, GetBlockBodiesMsg, GetCodeMsg, GetReceiptsMsg, GetProofsV1Msg, SendTxMsg, SendTxV2Msg, GetTxStatusMsg, GetHeaderProofsMsg, GetProofsV2Msg, GetHelperTrieProofsMsg} - // handleMsg is invoked whenever an inbound message is received from a remote // peer. The remote connection is torn down upon returning any error. func (pm *ProtocolManager) handleMsg(p *peer) error { @@ -364,20 +447,33 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } p.Log().Trace("Light Ethereum message arrived", "code", msg.Code, "bytes", msg.Size) - costs := p.fcCosts[msg.Code] - reject := func(reqCnt, maxCnt uint64) bool { + p.responseCount++ + responseCount := p.responseCount + var ( + maxCost uint64 + priority int64 + ) + + reject := func(reqID, reqCnt, maxCnt uint64) bool { + if reqCnt == 0 { + return true + } if p.fcClient == nil || reqCnt > maxCnt { return true } - bufValue, _ := p.fcClient.AcceptRequest() - cost := costs.baseCost + reqCnt*costs.reqCost - if cost > pm.server.defParams.BufLimit { - cost = pm.server.defParams.BufLimit + costs := p.fcCosts[msg.Code] + maxCost = costs.baseCost + reqCnt*costs.reqCost + if maxCost > p.fcParams.BufLimit { + maxCost = p.fcParams.BufLimit } - if cost > bufValue { - recharge := time.Duration((cost - bufValue) * 1000000 / pm.server.defParams.MinRecharge) - p.Log().Error("Request came too early", "recharge", common.PrettyDuration(recharge)) + + if accepted, bufShort, servingPriority := p.fcClient.AcceptRequest(reqID, responseCount, maxCost); !accepted { + if bufShort > 0 { + p.Log().Error("Request came too early", "remaining", common.PrettyDuration(time.Duration(bufShort*1000000/p.fcParams.MinRecharge))) + } return true + } else { + priority = servingPriority } return false } @@ -389,6 +485,41 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { var deliverMsg *Msg + errorFn := func(err error) { + if err != nil { + p.errCh <- err + } + } + + sendReq := func(reqID, amount uint64, reply *reply, servingTime uint64) { + p.responseLock.Lock() + defer p.responseLock.Unlock() + + realCost := servingTime + inSizeCost := uint64(float64(msg.Size) * pm.inSizeCostFactor) + if inSizeCost > realCost { + realCost = inSizeCost + } + if reply != nil { + outSizeCost := uint64(float64(reply.size()) * pm.outSizeCostFactor) + if outSizeCost > realCost { + realCost = outSizeCost + } + } + + bv := p.fcClient.RequestProcessed(reqID, responseCount, maxCost, realCost) + if pm.server.fcCostStats != nil { + pm.server.fcCostStats.update(msg.Code, amount, realCost) + } + if reply != nil { + p.queueSend(func() { + if err := reply.send(bv); err != nil { + p.errCh <- err + } + }) + } + } + // Handle the message depending on its contents switch msg.Code { case StatusMsg: @@ -440,7 +571,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } query := req.Query - if reject(query.Amount, MaxHeaderFetch) { + if reject(req.ReqID, query.Amount, MaxHeaderFetch) { return errResp(ErrRequestRejected, "") } @@ -454,79 +585,85 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { headers []*types.Header unknown bool ) - for !unknown && len(headers) < int(query.Amount) && bytes < softResponseLimit { - // Retrieve the next header satisfying the query - var origin *types.Header - if hashMode { - if first { - first = false - origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) - if origin != nil { - query.Origin.Number = origin.Number.Uint64() + + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + // Retrieve the next header satisfying the query + var origin *types.Header + if hashMode { + if first { + first = false + origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) + if origin != nil { + query.Origin.Number = origin.Number.Uint64() + } + } else { + origin = pm.blockchain.GetHeader(query.Origin.Hash, query.Origin.Number) } } else { - origin = pm.blockchain.GetHeader(query.Origin.Hash, query.Origin.Number) + origin = pm.blockchain.GetHeaderByNumber(query.Origin.Number) } - } else { - origin = pm.blockchain.GetHeaderByNumber(query.Origin.Number) - } - if origin == nil { - break - } - headers = append(headers, origin) - bytes += estHeaderRlpSize + if origin == nil { + return true, nil + } + headers = append(headers, origin) + bytes += estHeaderRlpSize - // Advance to the next header of the query - switch { - case hashMode && query.Reverse: - // Hash based traversal towards the genesis block - ancestor := query.Skip + 1 - if ancestor == 0 { - unknown = true - } else { - query.Origin.Hash, query.Origin.Number = pm.blockchain.GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) - unknown = (query.Origin.Hash == common.Hash{}) - } - case hashMode && !query.Reverse: - // Hash based traversal towards the leaf block - var ( - current = origin.Number.Uint64() - next = current + query.Skip + 1 - ) - if next <= current { - infos, _ := json.MarshalIndent(p.Peer.Info(), "", " ") - p.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", query.Skip, "next", next, "attacker", infos) - unknown = true - } else { - if header := pm.blockchain.GetHeaderByNumber(next); header != nil { - nextHash := header.Hash() - expOldHash, _ := pm.blockchain.GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) - if expOldHash == query.Origin.Hash { - query.Origin.Hash, query.Origin.Number = nextHash, next + // Advance to the next header of the query + switch { + case hashMode && query.Reverse: + // Hash based traversal towards the genesis block + ancestor := query.Skip + 1 + if ancestor == 0 { + unknown = true + } else { + query.Origin.Hash, query.Origin.Number = pm.blockchain.GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) + unknown = (query.Origin.Hash == common.Hash{}) + } + case hashMode && !query.Reverse: + // Hash based traversal towards the leaf block + var ( + current = origin.Number.Uint64() + next = current + query.Skip + 1 + ) + if next <= current { + infos, _ := json.MarshalIndent(p.Peer.Info(), "", " ") + p.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", query.Skip, "next", next, "attacker", infos) + unknown = true + } else { + if header := pm.blockchain.GetHeaderByNumber(next); header != nil { + nextHash := header.Hash() + expOldHash, _ := pm.blockchain.GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) + if expOldHash == query.Origin.Hash { + query.Origin.Hash, query.Origin.Number = nextHash, next + } else { + unknown = true + } } else { unknown = true } + } + case query.Reverse: + // Number based traversal towards the genesis block + if query.Origin.Number >= query.Skip+1 { + query.Origin.Number -= query.Skip + 1 } else { unknown = true } - } - case query.Reverse: - // Number based traversal towards the genesis block - if query.Origin.Number >= query.Skip+1 { - query.Origin.Number -= query.Skip + 1 - } else { - unknown = true - } - case !query.Reverse: - // Number based traversal towards the leaf block - query.Origin.Number += query.Skip + 1 - } - } - - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + query.Amount*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, query.Amount, rcost) - return p.SendBlockHeaders(req.ReqID, bv, headers) + case !query.Reverse: + // Number based traversal towards the leaf block + query.Origin.Number += query.Skip + 1 + } + return unknown || len(headers) >= int(query.Amount) || bytes >= softResponseLimit, nil + }, + // after: sendFunc(query.Amount, func(bv uint64) error { return p.SendBlockHeaders(req.ReqID, bv, headers) }), + send: func(servingTime uint64) { + sendReq(req.ReqID, query.Amount, p.ReplyBlockHeaders(req.ReqID, headers), servingTime) + }, + fail: errorFn, + }) case BlockHeadersMsg: if pm.downloader == nil { @@ -542,7 +679,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) if pm.fetcher != nil && pm.fetcher.requestedID(resp.ReqID) { pm.fetcher.deliverHeaders(p, resp.ReqID, resp.Headers) } else { @@ -568,24 +705,33 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { bodies []rlp.RawValue ) reqCnt := len(req.Hashes) - if reject(uint64(reqCnt), MaxBodyFetch) { + if reject(req.ReqID, uint64(reqCnt), MaxBodyFetch) { return errResp(ErrRequestRejected, "") } - for _, hash := range req.Hashes { - if bytes >= softResponseLimit { - break - } - // Retrieve the requested block body, stopping if enough was found - if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { - if data := rawdb.ReadBodyRLP(pm.chainDb, hash, *number); len(data) != 0 { - bodies = append(bodies, data) - bytes += len(data) + + index := 0 + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + hash := req.Hashes[index] + index++ + if bytes >= softResponseLimit { + return true, nil } - } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendBlockBodiesRLP(req.ReqID, bv, bodies) + // Retrieve the requested block body, stopping if enough was found + if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { + if data := rawdb.ReadBodyRLP(pm.chainDb, hash, *number); len(data) != 0 { + bodies = append(bodies, data) + bytes += len(data) + } + } + return index == reqCnt, nil + }, + send: func(servingTime uint64) { + sendReq(req.ReqID, uint64(reqCnt), p.ReplyBlockBodiesRLP(req.ReqID, bodies), servingTime) + }, + fail: errorFn, + }) case BlockBodiesMsg: if pm.odr == nil { @@ -601,7 +747,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgBlockBodies, ReqID: resp.ReqID, @@ -624,33 +770,42 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { data [][]byte ) reqCnt := len(req.Reqs) - if reject(uint64(reqCnt), MaxCodeFetch) { + if reject(req.ReqID, uint64(reqCnt), MaxCodeFetch) { return errResp(ErrRequestRejected, "") } - for _, req := range req.Reqs { - // Retrieve the requested state entry, stopping if enough was found - if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { - if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { - statedb, err := pm.blockchain.State() - if err != nil { - continue - } - account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) - if err != nil { - continue - } - code, _ := statedb.Database().TrieDB().Node(common.BytesToHash(account.CodeHash)) + index := 0 + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + req := req.Reqs[index] + index++ + done := index == reqCnt + // Retrieve the requested state entry, stopping if enough was found + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, err := pm.blockchain.State() + if err != nil { + return done, nil + } + account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + if err != nil { + return done, nil + } + code, _ := statedb.Database().TrieDB().Node(common.BytesToHash(account.CodeHash)) - data = append(data, code) - if bytes += len(code); bytes >= softResponseLimit { - break + data = append(data, code) + if bytes += len(code); bytes >= softResponseLimit { + return true, nil + } } } - } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendCode(req.ReqID, bv, data) + return done, nil + }, + send: func(servingTime uint64) { + sendReq(req.ReqID, uint64(reqCnt), p.ReplyCode(req.ReqID, data), servingTime) + }, + fail: errorFn, + }) case CodeMsg: if pm.odr == nil { @@ -666,7 +821,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgCode, ReqID: resp.ReqID, @@ -689,34 +844,44 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { receipts []rlp.RawValue ) reqCnt := len(req.Hashes) - if reject(uint64(reqCnt), MaxReceiptFetch) { + if reject(req.ReqID, uint64(reqCnt), MaxReceiptFetch) { return errResp(ErrRequestRejected, "") } - for _, hash := range req.Hashes { - if bytes >= softResponseLimit { - break - } - // Retrieve the requested block's receipts, skipping if unknown to us - var results types.Receipts - if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { - results = rawdb.ReadReceipts(pm.chainDb, hash, *number) - } - if results == nil { - if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { - continue + + index := 0 + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + hash := req.Hashes[index] + index++ + done := index == reqCnt + if bytes >= softResponseLimit { + return true, nil } - } - // If known, encode and queue for response packet - if encoded, err := rlp.EncodeToBytes(results); err != nil { - log.Error("Failed to encode receipt", "err", err) - } else { - receipts = append(receipts, encoded) - bytes += len(encoded) - } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendReceiptsRLP(req.ReqID, bv, receipts) + // Retrieve the requested block's receipts, skipping if unknown to us + var results types.Receipts + if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { + results = rawdb.ReadReceipts(pm.chainDb, hash, *number) + } + if results == nil { + if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { + return done, nil + } + } + // If known, encode and queue for response packet + if encoded, err := rlp.EncodeToBytes(results); err != nil { + log.Error("Failed to encode receipt", "err", err) + } else { + receipts = append(receipts, encoded) + bytes += len(encoded) + } + return done, nil + }, + send: func(servingTime uint64) { + sendReq(req.ReqID, uint64(reqCnt), p.ReplyReceiptsRLP(req.ReqID, receipts), servingTime) + }, + fail: errorFn, + }) case ReceiptsMsg: if pm.odr == nil { @@ -732,7 +897,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgReceipts, ReqID: resp.ReqID, @@ -755,42 +920,52 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { proofs proofsData ) reqCnt := len(req.Reqs) - if reject(uint64(reqCnt), MaxProofsFetch) { + if reject(req.ReqID, uint64(reqCnt), MaxProofsFetch) { return errResp(ErrRequestRejected, "") } - for _, req := range req.Reqs { - // Retrieve the requested state entry, stopping if enough was found - if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { - if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { - statedb, err := pm.blockchain.State() - if err != nil { - continue - } - var trie state.Trie - if len(req.AccKey) > 0 { - account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) - if err != nil { - continue - } - trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) - } else { - trie, _ = statedb.Database().OpenTrie(header.Root) - } - if trie != nil { - var proof light.NodeList - trie.Prove(req.Key, 0, &proof) - proofs = append(proofs, proof) - if bytes += proof.DataSize(); bytes >= softResponseLimit { - break + index := 0 + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + req := req.Reqs[index] + index++ + done := index == reqCnt + // Retrieve the requested state entry, stopping if enough was found + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, err := pm.blockchain.State() + if err != nil { + return done, nil + } + var trie state.Trie + if len(req.AccKey) > 0 { + account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + if err != nil { + return done, nil + } + trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) + } else { + trie, _ = statedb.Database().OpenTrie(header.Root) + } + if trie != nil { + var proof light.NodeList + trie.Prove(req.Key, 0, &proof) + + proofs = append(proofs, proof) + if bytes += proof.DataSize(); bytes >= softResponseLimit { + return true, nil + } } } } - } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendProofs(req.ReqID, bv, proofs) + return done, nil + }, + send: func(servingTime uint64) { + sendReq(req.ReqID, uint64(reqCnt), p.ReplyProofs(req.ReqID, proofs), servingTime) + }, + fail: errorFn, + }) case GetProofsV2Msg: p.Log().Trace("Received les/2 proofs request") @@ -809,50 +984,59 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { root common.Hash ) reqCnt := len(req.Reqs) - if reject(uint64(reqCnt), MaxProofsFetch) { + if reject(req.ReqID, uint64(reqCnt), MaxProofsFetch) { return errResp(ErrRequestRejected, "") } nodes := light.NewNodeSet() - for _, req := range req.Reqs { - // Look up the state belonging to the request - if statedb == nil || req.BHash != lastBHash { - statedb, root, lastBHash = nil, common.Hash{}, req.BHash + index := 0 + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + req := req.Reqs[index] + index++ + done := index == reqCnt + // Look up the state belonging to the request + if statedb == nil || req.BHash != lastBHash { + statedb, root, lastBHash = nil, common.Hash{}, req.BHash - if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { - if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { - statedb, _ = pm.blockchain.State() - root = header.Root + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, _ = pm.blockchain.State() + root = header.Root + } } } - } - if statedb == nil { - continue - } - // Pull the account or storage trie of the request - var trie state.Trie - if len(req.AccKey) > 0 { - account, err := pm.getAccount(statedb, root, common.BytesToHash(req.AccKey)) - if err != nil { - continue + if statedb == nil { + return done, nil } - trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) - } else { - trie, _ = statedb.Database().OpenTrie(root) - } - if trie == nil { - continue - } - // Prove the user's request from the account or stroage trie - trie.Prove(req.Key, req.FromLevel, nodes) - if nodes.DataSize() >= softResponseLimit { - break - } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendProofsV2(req.ReqID, bv, nodes.NodeList()) + // Pull the account or storage trie of the request + var trie state.Trie + if len(req.AccKey) > 0 { + account, err := pm.getAccount(statedb, root, common.BytesToHash(req.AccKey)) + if err != nil { + return done, nil + } + trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) + } else { + trie, _ = statedb.Database().OpenTrie(root) + } + if trie == nil { + return done, nil + } + // Prove the user's request from the account or stroage trie + trie.Prove(req.Key, req.FromLevel, nodes) + if nodes.DataSize() >= softResponseLimit { + return true, nil + } + return done, nil + }, + send: func(servingTime uint64) { + sendReq(req.ReqID, uint64(reqCnt), p.ReplyProofsV2(req.ReqID, nodes.NodeList()), servingTime) + }, + fail: errorFn, + }) case ProofsV1Msg: if pm.odr == nil { @@ -868,7 +1052,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgProofsV1, ReqID: resp.ReqID, @@ -889,7 +1073,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgProofsV2, ReqID: resp.ReqID, @@ -912,34 +1096,44 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { proofs []ChtResp ) reqCnt := len(req.Reqs) - if reject(uint64(reqCnt), MaxHelperTrieProofsFetch) { + if reject(req.ReqID, uint64(reqCnt), MaxHelperTrieProofsFetch) { return errResp(ErrRequestRejected, "") } trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) - for _, req := range req.Reqs { - if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { - sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*pm.iConfig.ChtSize-1) - if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { - trie, err := trie.New(root, trieDb) - if err != nil { - continue - } - var encNumber [8]byte - binary.BigEndian.PutUint64(encNumber[:], req.BlockNum) - var proof light.NodeList - trie.Prove(encNumber[:], 0, &proof) + index := 0 + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + req := req.Reqs[index] + index++ + done := index == reqCnt + if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*pm.iConfig.ChtSize-1) + if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { + trie, err := trie.New(root, trieDb) + if err != nil { + return done, nil + } + var encNumber [8]byte + binary.BigEndian.PutUint64(encNumber[:], req.BlockNum) - proofs = append(proofs, ChtResp{Header: header, Proof: proof}) - if bytes += proof.DataSize() + estHeaderRlpSize; bytes >= softResponseLimit { - break + var proof light.NodeList + trie.Prove(encNumber[:], 0, &proof) + + proofs = append(proofs, ChtResp{Header: header, Proof: proof}) + if bytes += proof.DataSize() + estHeaderRlpSize; bytes >= softResponseLimit { + return true, nil + } } } - } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendHeaderProofs(req.ReqID, bv, proofs) + return done, nil + }, + send: func(servingTime uint64) { + sendReq(req.ReqID, uint64(reqCnt), p.ReplyHeaderProofs(req.ReqID, proofs), servingTime) + }, + fail: errorFn, + }) case GetHelperTrieProofsMsg: p.Log().Trace("Received helper trie proof request") @@ -957,7 +1151,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { auxData [][]byte ) reqCnt := len(req.Reqs) - if reject(uint64(reqCnt), MaxHelperTrieProofsFetch) { + if reject(req.ReqID, uint64(reqCnt), MaxHelperTrieProofsFetch) { return errResp(ErrRequestRejected, "") } @@ -968,39 +1162,48 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { auxTrie *trie.Trie ) nodes := light.NewNodeSet() - for _, req := range req.Reqs { - if auxTrie == nil || req.Type != lastType || req.TrieIdx != lastIdx { - auxTrie, lastType, lastIdx = nil, req.Type, req.TrieIdx - var prefix string - if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) { - auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix))) + index := 0 + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + req := req.Reqs[index] + index++ + if auxTrie == nil || req.Type != lastType || req.TrieIdx != lastIdx { + auxTrie, lastType, lastIdx = nil, req.Type, req.TrieIdx + + var prefix string + if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) { + auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix))) + } } - } - if req.AuxReq == auxRoot { - var data []byte - if root != (common.Hash{}) { - data = root[:] - } - auxData = append(auxData, data) - auxBytes += len(data) - } else { - if auxTrie != nil { - auxTrie.Prove(req.Key, req.FromLevel, nodes) - } - if req.AuxReq != 0 { - data := pm.getHelperTrieAuxData(req) + if req.AuxReq == auxRoot { + var data []byte + if root != (common.Hash{}) { + data = root[:] + } auxData = append(auxData, data) auxBytes += len(data) + } else { + if auxTrie != nil { + auxTrie.Prove(req.Key, req.FromLevel, nodes) + } + if req.AuxReq != 0 { + data := pm.getHelperTrieAuxData(req) + auxData = append(auxData, data) + auxBytes += len(data) + } } - } - if nodes.DataSize()+auxBytes >= softResponseLimit { - break - } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData}) + if nodes.DataSize()+auxBytes >= softResponseLimit { + return true, nil + } + return index == reqCnt, nil + }, + send: func(servingTime uint64) { + sendReq(req.ReqID, uint64(reqCnt), p.ReplyHelperTrieProofs(req.ReqID, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData}), servingTime) + }, + fail: errorFn, + }) case HeaderProofsMsg: if pm.odr == nil { @@ -1015,7 +1218,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgHeaderProofs, ReqID: resp.ReqID, @@ -1036,7 +1239,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgHelperTrieProofs, ReqID: resp.ReqID, @@ -1053,13 +1256,21 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } reqCnt := len(txs) - if reject(uint64(reqCnt), MaxTxSend) { + if reject(0, uint64(reqCnt), MaxTxSend) { return errResp(ErrRequestRejected, "") } - pm.txpool.AddRemotes(txs) - _, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + pm.txpool.AddRemotes(txs) + return true, nil + }, + send: func(servingTime uint64) { + sendReq(0, uint64(reqCnt), nil, servingTime) + }, + fail: errorFn, + }) case SendTxV2Msg: if pm.txpool == nil { @@ -1074,29 +1285,35 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } reqCnt := len(req.Txs) - if reject(uint64(reqCnt), MaxTxSend) { + if reject(req.ReqID, uint64(reqCnt), MaxTxSend) { return errResp(ErrRequestRejected, "") } - hashes := make([]common.Hash, len(req.Txs)) - for i, tx := range req.Txs { - hashes[i] = tx.Hash() - } - stats := pm.txStatus(hashes) - for i, stat := range stats { - if stat.Status == core.TxStatusUnknown { - if errs := pm.txpool.AddRemotes([]*types.Transaction{req.Txs[i]}); errs[0] != nil { - stats[i].Error = errs[0].Error() - continue + var stats []txStatus + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + hashes := make([]common.Hash, len(req.Txs)) + for i, tx := range req.Txs { + hashes[i] = tx.Hash() } - stats[i] = pm.txStatus([]common.Hash{hashes[i]})[0] - } - } - - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - - return p.SendTxStatus(req.ReqID, bv, stats) + stats = pm.txStatus(hashes) + for i, stat := range stats { + if stat.Status == core.TxStatusUnknown { + if errs := pm.txpool.AddRemotes([]*types.Transaction{req.Txs[i]}); errs[0] != nil { + stats[i].Error = errs[0].Error() + continue + } + stats[i] = pm.txStatus([]common.Hash{hashes[i]})[0] + } + } + return true, nil + }, + send: func(servingTime uint64) { + sendReq(req.ReqID, uint64(reqCnt), p.ReplyTxStatus(req.ReqID, stats), servingTime) + }, + fail: errorFn, + }) case GetTxStatusMsg: if pm.txpool == nil { @@ -1111,13 +1328,22 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } reqCnt := len(req.Hashes) - if reject(uint64(reqCnt), MaxTxStatus) { + if reject(req.ReqID, uint64(reqCnt), MaxTxStatus) { return errResp(ErrRequestRejected, "") } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendTxStatus(req.ReqID, bv, pm.txStatus(req.Hashes)) + var stats []txStatus + pm.servingQueue.addTask(&servingTask{ + priority: priority, + run: func() (bool, error) { + stats = pm.txStatus(req.Hashes) + return true, nil + }, + send: func(servingTime uint64) { + sendReq(req.ReqID, uint64(reqCnt), p.ReplyTxStatus(req.ReqID, stats), servingTime) + }, + fail: errorFn, + }) case TxStatusMsg: if pm.odr == nil { @@ -1133,7 +1359,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) default: p.Log().Trace("Received unknown message", "code", msg.Code) @@ -1243,7 +1469,7 @@ func (pc *peerConnection) RequestHeadersByHash(origin common.Hash, amount int, s request: func(dp distPeer) func() { peer := dp.(*peer) cost := peer.GetRequestCost(GetBlockHeadersMsg, amount) - peer.fcServer.QueueRequest(reqID, cost) + peer.fcServer.QueuedRequest(reqID, cost) return func() { peer.RequestHeadersByHash(reqID, cost, origin, amount, skip, reverse) } }, } @@ -1267,7 +1493,7 @@ func (pc *peerConnection) RequestHeadersByNumber(origin uint64, amount int, skip request: func(dp distPeer) func() { peer := dp.(*peer) cost := peer.GetRequestCost(GetBlockHeadersMsg, amount) - peer.fcServer.QueueRequest(reqID, cost) + peer.fcServer.QueuedRequest(reqID, cost) return func() { peer.RequestHeadersByNumber(reqID, cost, origin, amount, skip, reverse) } }, } diff --git a/les/helper_test.go b/les/helper_test.go index 777a6bf6be..7434b38e90 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -134,9 +135,9 @@ func testIndexers(db ethdb.Database, odr light.OdrBackend, iConfig *light.Indexe } func testRCL() RequestCostList { - cl := make(RequestCostList, len(reqList)) - for i, code := range reqList { - cl[i].MsgCode = code + cl := make(RequestCostList, len(reqBenchMap)) + for i, req := range reqBenchMap { + cl[i].MsgCode = req.code cl[i].BaseCost = 0 cl[i].ReqCost = 0 } @@ -183,14 +184,16 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor if !lightSync { srv := &LesServer{lesCommons: lesCommons{protocolManager: pm}} pm.server = srv + pm.servingQueue.setThreads(4) srv.defParams = flowcontrol.ServerParams{ BufLimit: testBufLimit, MinRecharge: 1, } - srv.fcManager = flowcontrol.NewClientManager(50, 10, 1000000000) - srv.fcCostStats = newCostStats(nil) + srv.fcManager = flowcontrol.NewClientManager(nil, &mclock.System{}) + srv.fcCostList = testRCL() + srv.fcCostTable = srv.fcCostList.decode() } pm.Start(1000) return pm, nil @@ -313,7 +316,7 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNu t.Fatalf("status send: %v", err) } - p.fcServerParams = flowcontrol.ServerParams{ + p.fcParams = flowcontrol.ServerParams{ BufLimit: testBufLimit, MinRecharge: 1, } @@ -375,7 +378,7 @@ func newClientServerEnv(t *testing.T, blocks int, protocol int, waitIndexers fun db, ldb := ethdb.NewMemDatabase(), ethdb.NewMemDatabase() peers, lPeers := newPeerSet(), newPeerSet() - dist := newRequestDistributor(lPeers, make(chan struct{})) + dist := newRequestDistributor(lPeers, make(chan struct{}), &mclock.System{}) rm := newRetrieveManager(lPeers, dist, nil) odr := NewLesOdr(ldb, light.TestClientIndexerConfig, rm) diff --git a/les/odr.go b/les/odr.go index f7592354de..5d98c66a90 100644 --- a/les/odr.go +++ b/les/odr.go @@ -117,7 +117,7 @@ func (odr *LesOdr) Retrieve(ctx context.Context, req light.OdrRequest) (err erro request: func(dp distPeer) func() { p := dp.(*peer) cost := lreq.GetCost(p) - p.fcServer.QueueRequest(reqID, cost) + p.fcServer.QueuedRequest(reqID, cost) return func() { lreq.Request(reqID, p) } }, } diff --git a/les/peer.go b/les/peer.go index e16d731c5c..5deadd785f 100644 --- a/les/peer.go +++ b/les/peer.go @@ -75,8 +75,13 @@ type peer struct { headInfo *announceData lock sync.RWMutex - announceChn chan announceData - sendQueue *execQueue + sendQueue *execQueue + + errCh chan error + // responseLock ensures that responses are queued in the same order as + // RequestProcessed is called + responseLock sync.Mutex + responseCount uint64 poolEntry *poolEntry hasBlock func(common.Hash, uint64, bool) bool @@ -84,10 +89,10 @@ type peer struct { updateCounter uint64 updateTime mclock.AbsTime - fcClient *flowcontrol.ClientNode // nil if the peer is server only - fcServer *flowcontrol.ServerNode // nil if the peer is client only - fcServerParams flowcontrol.ServerParams - fcCosts requestCostTable + fcClient *flowcontrol.ClientNode // nil if the peer is server only + fcServer *flowcontrol.ServerNode // nil if the peer is client only + fcParams flowcontrol.ServerParams + fcCosts requestCostTable isTrusted bool isOnlyAnnounce bool @@ -97,13 +102,12 @@ func newPeer(version int, network uint64, isTrusted bool, p *p2p.Peer, rw p2p.Ms id := p.ID() return &peer{ - Peer: p, - rw: rw, - version: version, - network: network, - id: fmt.Sprintf("%x", id[:8]), - announceChn: make(chan announceData, 20), - isTrusted: isTrusted, + Peer: p, + rw: rw, + version: version, + network: network, + id: fmt.Sprintf("%x", id[:8]), + isTrusted: isTrusted, } } @@ -182,6 +186,20 @@ func (p *peer) waitBefore(maxCost uint64) (time.Duration, float64) { return p.fcServer.CanSend(maxCost) } +// updateBandwidth updates the request serving bandwidth assigned to a given client +// and also sends an announcement about the updated flow control parameters +func (p *peer) updateBandwidth(bw uint64) { + p.responseLock.Lock() + defer p.responseLock.Unlock() + + p.fcParams = flowcontrol.ServerParams{MinRecharge: bw, BufLimit: bw * bufLimitRatio} + p.fcClient.UpdateParams(p.fcParams) + var kvList keyValueList + kvList = kvList.add("flowControl/MRR", bw) + kvList = kvList.add("flowControl/BL", bw*bufLimitRatio) + p.queueSend(func() { p.SendAnnounce(announceData{Update: kvList}) }) +} + func sendRequest(w p2p.MsgWriter, msgcode, reqID, cost uint64, data interface{}) error { type req struct { ReqID uint64 @@ -190,12 +208,27 @@ func sendRequest(w p2p.MsgWriter, msgcode, reqID, cost uint64, data interface{}) return p2p.Send(w, msgcode, req{reqID, data}) } -func sendResponse(w p2p.MsgWriter, msgcode, reqID, bv uint64, data interface{}) error { +// reply struct represents a reply with the actual data already RLP encoded and +// only the bv (buffer value) missing. This allows the serving mechanism to +// calculate the bv value which depends on the data size before sending the reply. +type reply struct { + w p2p.MsgWriter + msgcode, reqID uint64 + data rlp.RawValue +} + +// send sends the reply with the calculated buffer value +func (r *reply) send(bv uint64) error { type resp struct { ReqID, BV uint64 - Data interface{} + Data rlp.RawValue } - return p2p.Send(w, msgcode, resp{reqID, bv, data}) + return p2p.Send(r.w, r.msgcode, resp{r.reqID, bv, r.data}) +} + +// size returns the RLP encoded size of the message data +func (r *reply) size() uint32 { + return uint32(len(r.data)) } func (p *peer) GetRequestCost(msgcode uint64, amount int) uint64 { @@ -203,8 +236,8 @@ func (p *peer) GetRequestCost(msgcode uint64, amount int) uint64 { defer p.lock.RUnlock() cost := p.fcCosts[msgcode].baseCost + p.fcCosts[msgcode].reqCost*uint64(amount) - if cost > p.fcServerParams.BufLimit { - cost = p.fcServerParams.BufLimit + if cost > p.fcParams.BufLimit { + cost = p.fcParams.BufLimit } return cost } @@ -229,8 +262,8 @@ func (p *peer) GetTxRelayCost(amount, size int) uint64 { cost = sizeCost } - if cost > p.fcServerParams.BufLimit { - cost = p.fcServerParams.BufLimit + if cost > p.fcParams.BufLimit { + cost = p.fcParams.BufLimit } return cost } @@ -249,52 +282,61 @@ func (p *peer) SendAnnounce(request announceData) error { return p2p.Send(p.rw, AnnounceMsg, request) } -// SendBlockHeaders sends a batch of block headers to the remote peer. -func (p *peer) SendBlockHeaders(reqID, bv uint64, headers []*types.Header) error { - return sendResponse(p.rw, BlockHeadersMsg, reqID, bv, headers) +// ReplyBlockHeaders creates a reply with a batch of block headers +func (p *peer) ReplyBlockHeaders(reqID uint64, headers []*types.Header) *reply { + data, _ := rlp.EncodeToBytes(headers) + return &reply{p.rw, BlockHeadersMsg, reqID, data} } -// SendBlockBodiesRLP sends a batch of block contents to the remote peer from +// ReplyBlockBodiesRLP creates a reply with a batch of block contents from // an already RLP encoded format. -func (p *peer) SendBlockBodiesRLP(reqID, bv uint64, bodies []rlp.RawValue) error { - return sendResponse(p.rw, BlockBodiesMsg, reqID, bv, bodies) +func (p *peer) ReplyBlockBodiesRLP(reqID uint64, bodies []rlp.RawValue) *reply { + data, _ := rlp.EncodeToBytes(bodies) + return &reply{p.rw, BlockBodiesMsg, reqID, data} } -// SendCodeRLP sends a batch of arbitrary internal data, corresponding to the +// ReplyCode creates a reply with a batch of arbitrary internal data, corresponding to the // hashes requested. -func (p *peer) SendCode(reqID, bv uint64, data [][]byte) error { - return sendResponse(p.rw, CodeMsg, reqID, bv, data) +func (p *peer) ReplyCode(reqID uint64, codes [][]byte) *reply { + data, _ := rlp.EncodeToBytes(codes) + return &reply{p.rw, CodeMsg, reqID, data} } -// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the +// ReplyReceiptsRLP creates a reply with a batch of transaction receipts, corresponding to the // ones requested from an already RLP encoded format. -func (p *peer) SendReceiptsRLP(reqID, bv uint64, receipts []rlp.RawValue) error { - return sendResponse(p.rw, ReceiptsMsg, reqID, bv, receipts) +func (p *peer) ReplyReceiptsRLP(reqID uint64, receipts []rlp.RawValue) *reply { + data, _ := rlp.EncodeToBytes(receipts) + return &reply{p.rw, ReceiptsMsg, reqID, data} } -// SendProofs sends a batch of legacy LES/1 merkle proofs, corresponding to the ones requested. -func (p *peer) SendProofs(reqID, bv uint64, proofs proofsData) error { - return sendResponse(p.rw, ProofsV1Msg, reqID, bv, proofs) +// ReplyProofs creates a reply with a batch of legacy LES/1 merkle proofs, corresponding to the ones requested. +func (p *peer) ReplyProofs(reqID uint64, proofs proofsData) *reply { + data, _ := rlp.EncodeToBytes(proofs) + return &reply{p.rw, ProofsV1Msg, reqID, data} } -// SendProofsV2 sends a batch of merkle proofs, corresponding to the ones requested. -func (p *peer) SendProofsV2(reqID, bv uint64, proofs light.NodeList) error { - return sendResponse(p.rw, ProofsV2Msg, reqID, bv, proofs) +// ReplyProofsV2 creates a reply with a batch of merkle proofs, corresponding to the ones requested. +func (p *peer) ReplyProofsV2(reqID uint64, proofs light.NodeList) *reply { + data, _ := rlp.EncodeToBytes(proofs) + return &reply{p.rw, ProofsV2Msg, reqID, data} } -// SendHeaderProofs sends a batch of legacy LES/1 header proofs, corresponding to the ones requested. -func (p *peer) SendHeaderProofs(reqID, bv uint64, proofs []ChtResp) error { - return sendResponse(p.rw, HeaderProofsMsg, reqID, bv, proofs) +// ReplyHeaderProofs creates a reply with a batch of legacy LES/1 header proofs, corresponding to the ones requested. +func (p *peer) ReplyHeaderProofs(reqID uint64, proofs []ChtResp) *reply { + data, _ := rlp.EncodeToBytes(proofs) + return &reply{p.rw, HeaderProofsMsg, reqID, data} } -// SendHelperTrieProofs sends a batch of HelperTrie proofs, corresponding to the ones requested. -func (p *peer) SendHelperTrieProofs(reqID, bv uint64, resp HelperTrieResps) error { - return sendResponse(p.rw, HelperTrieProofsMsg, reqID, bv, resp) +// ReplyHelperTrieProofs creates a reply with a batch of HelperTrie proofs, corresponding to the ones requested. +func (p *peer) ReplyHelperTrieProofs(reqID uint64, resp HelperTrieResps) *reply { + data, _ := rlp.EncodeToBytes(resp) + return &reply{p.rw, HelperTrieProofsMsg, reqID, data} } -// SendTxStatus sends a batch of transaction status records, corresponding to the ones requested. -func (p *peer) SendTxStatus(reqID, bv uint64, stats []txStatus) error { - return sendResponse(p.rw, TxStatusMsg, reqID, bv, stats) +// ReplyTxStatus creates a reply with a batch of transaction status records, corresponding to the ones requested. +func (p *peer) ReplyTxStatus(reqID uint64, stats []txStatus) *reply { + data, _ := rlp.EncodeToBytes(stats) + return &reply{p.rw, TxStatusMsg, reqID, data} } // RequestHeadersByHash fetches a batch of blocks' headers corresponding to the @@ -372,7 +414,7 @@ func (p *peer) RequestTxStatus(reqID, cost uint64, txHashes []common.Hash) error return sendRequest(p.rw, GetTxStatusMsg, reqID, cost, txHashes) } -// SendTxStatus sends a batch of transactions to be added to the remote transaction pool. +// SendTxStatus creates a reply with a batch of transactions to be added to the remote transaction pool. func (p *peer) SendTxs(reqID, cost uint64, txs rlp.RawValue) error { p.Log().Debug("Sending batch of transactions", "size", len(txs)) switch p.version { @@ -477,9 +519,9 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis } send = send.add("flowControl/BL", server.defParams.BufLimit) send = send.add("flowControl/MRR", server.defParams.MinRecharge) - list := server.fcCostStats.getCurrentList() - send = send.add("flowControl/MRC", list) - p.fcCosts = list.decode() + send = send.add("flowControl/MRC", server.fcCostList) + p.fcCosts = server.fcCostTable + p.fcParams = server.defParams } else { //on client node p.announceType = announceTypeSimple @@ -568,8 +610,8 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis if err := recv.get("flowControl/MRC", &MRC); err != nil { return err } - p.fcServerParams = params - p.fcServer = flowcontrol.NewServerNode(params) + p.fcParams = params + p.fcServer = flowcontrol.NewServerNode(params, &mclock.System{}) p.fcCosts = MRC.decode() } p.headInfo = &announceData{Td: rTd, Hash: rHash, Number: rNum} @@ -582,7 +624,7 @@ func (p *peer) updateFlowControl(update keyValueMap) { if p.fcServer == nil { return } - params := p.fcServerParams + params := p.fcParams updateParams := false if update.get("flowControl/BL", ¶ms.BufLimit) == nil { updateParams = true @@ -591,7 +633,7 @@ func (p *peer) updateFlowControl(update keyValueMap) { updateParams = true } if updateParams { - p.fcServerParams = params + p.fcParams = params p.fcServer.UpdateParams(params) } var MRC RequestCostList diff --git a/les/protocol.go b/les/protocol.go index 8db23c10d7..992ecb256d 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -81,6 +81,25 @@ const ( TxStatusMsg = 0x15 ) +type requestInfo struct { + name string + maxCount uint64 +} + +var requests = map[uint64]requestInfo{ + GetBlockHeadersMsg: {"GetBlockHeaders", MaxHeaderFetch}, + GetBlockBodiesMsg: {"GetBlockBodies", MaxBodyFetch}, + GetReceiptsMsg: {"GetReceipts", MaxReceiptFetch}, + GetProofsV1Msg: {"GetProofsV1", MaxProofsFetch}, + GetCodeMsg: {"GetCode", MaxCodeFetch}, + SendTxMsg: {"SendTx", MaxTxSend}, + GetHeaderProofsMsg: {"GetHeaderProofs", MaxHelperTrieProofsFetch}, + GetProofsV2Msg: {"GetProofsV2", MaxProofsFetch}, + GetHelperTrieProofsMsg: {"GetHelperTrieProofs", MaxHelperTrieProofsFetch}, + SendTxV2Msg: {"SendTxV2", MaxTxSend}, + GetTxStatusMsg: {"GetTxStatus", MaxTxStatus}, +} + type errCode int const ( diff --git a/les/server.go b/les/server.go index eef027b3cf..20d5bb5022 100644 --- a/les/server.go +++ b/les/server.go @@ -19,35 +19,44 @@ package les import ( "crypto/ecdsa" - "encoding/binary" - "math" "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" +) + +const ( + bufLimitRatio = 6000 // fixed bufLimit/MRR ratio + makeCostStats = false // make request cost statistics during operation ) type LesServer struct { lesCommons fcManager *flowcontrol.ClientManager // nil if our node is client only + fcCostList RequestCostList + fcCostTable requestCostTable fcCostStats *requestCostStats defParams flowcontrol.ServerParams lesTopics []discv5.Topic privateKey *ecdsa.PrivateKey quitSync chan struct{} onlyAnnounce bool + + totalBandwidth, minBandwidth, minBufLimit, bufLimitRatio uint64 + bwcNormal, bwcBlockProcessing flowcontrol.PieceWiseLinear // bandwidth curve for normal operation and block processing mode + thcNormal, thcBlockProcessing int // serving thread count for normal operation and block processing mode } func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { @@ -93,6 +102,41 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { } logger := log.New() + pm.server = srv + + bwNormal := uint64(config.LightServ) * flowcontrol.FixedPointMultiplier / 100 + srv.bwcNormal = flowcontrol.PieceWiseLinear{{0, 0} /*{bwNormal / 10, bwNormal}, */, {bwNormal, bwNormal}} + // limit the serving thread count to at least 4 times the targeted average + // bandwidth, allowing more paralellization in short-term load spikes but + // still limiting the total thread count at a reasonable level + srv.thcNormal = int(bwNormal * 4 / flowcontrol.FixedPointMultiplier) + if srv.thcNormal < 4 { + srv.thcNormal = 4 + } + // while processing blocks use half of the normal target bandwidth + bwBlockProcessing := bwNormal / 2 + srv.bwcBlockProcessing = flowcontrol.PieceWiseLinear{{0, 0} /*{bwBlockProcessing / 10, bwBlockProcessing}, */, {bwBlockProcessing, bwBlockProcessing}} + // limit the serving thread count just above the targeted average bandwidth, + // ensuring that block processing is minimally hindered + srv.thcBlockProcessing = int(bwBlockProcessing/flowcontrol.FixedPointMultiplier) + 1 + + pm.servingQueue.setThreads(srv.thcNormal) + srv.fcManager = flowcontrol.NewClientManager(srv.bwcNormal, &mclock.System{}) + + srv.totalBandwidth = bwNormal + if config.LightBandwidthIn > 0 { + pm.inSizeCostFactor = float64(srv.totalBandwidth) / float64(config.LightBandwidthIn) + } + if config.LightBandwidthOut > 0 { + pm.outSizeCostFactor = float64(srv.totalBandwidth) / float64(config.LightBandwidthOut) + } + srv.fcCostList, srv.minBufLimit = pm.benchmarkCosts(srv.thcNormal, pm.inSizeCostFactor, pm.outSizeCostFactor) + srv.fcCostTable = srv.fcCostList.decode() + if makeCostStats { + srv.fcCostStats = newCostStats(srv.fcCostTable) + } + + srv.minBandwidth = (srv.minBufLimit-1)/bufLimitRatio + 1 chtV1SectionCount, _, _ := srv.chtIndexer.Sections() // indexer still uses LES/1 4k section size for backwards server compatibility chtV2SectionCount := chtV1SectionCount / (params.CHTFrequencyClient / params.CHTFrequencyServer) @@ -114,17 +158,44 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { } srv.chtIndexer.Start(eth.BlockChain()) - pm.server = srv - - srv.defParams = flowcontrol.ServerParams{ - BufLimit: 300000000, - MinRecharge: 50000, - } - srv.fcManager = flowcontrol.NewClientManager(uint64(config.LightServ), 10, 1000000000) - srv.fcCostStats = newCostStats(eth.ChainDb()) + srv.blockProcLoop(pm) return srv, nil } +func (s *LesServer) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: "les", + Version: "1.0", + Service: NewPrivateLesServerAPI(s), + Public: false, + }, + } +} + +func (s *LesServer) blockProcLoop(pm *ProtocolManager) { + pm.wg.Add(1) + procFeedback := make(chan bool, 10) + pm.blockchain.(*core.BlockChain).SetProcFeedback(procFeedback) + go func() { + for { + select { + case processing := <-procFeedback: + if processing { + pm.servingQueue.setThreads(s.thcBlockProcessing) + s.fcManager.SetRechargeCurve(s.bwcBlockProcessing) + } else { + pm.servingQueue.setThreads(s.thcNormal) + s.fcManager.SetRechargeCurve(s.bwcNormal) + } + case <-pm.quitSync: + pm.wg.Done() + return + } + } + }() +} + func (s *LesServer) Protocols() []p2p.Protocol { return s.makeProtocols(ServerProtocolVersions) } @@ -156,8 +227,9 @@ func (s *LesServer) SetBloomBitsIndexer(bloomIndexer *core.ChainIndexer) { func (s *LesServer) Stop() { s.chtIndexer.Close() // bloom trie indexer is closed by parent bloombits indexer - s.fcCostStats.store() - s.fcManager.Stop() + if s.fcCostStats != nil { + s.fcCostStats.printStats() + } go func() { <-s.protocolManager.noMorePeers }() @@ -185,156 +257,6 @@ func (list RequestCostList) decode() requestCostTable { return table } -type linReg struct { - sumX, sumY, sumXX, sumXY float64 - cnt uint64 -} - -const linRegMaxCnt = 100000 - -func (l *linReg) add(x, y float64) { - if l.cnt >= linRegMaxCnt { - sub := float64(l.cnt+1-linRegMaxCnt) / linRegMaxCnt - l.sumX -= l.sumX * sub - l.sumY -= l.sumY * sub - l.sumXX -= l.sumXX * sub - l.sumXY -= l.sumXY * sub - l.cnt = linRegMaxCnt - 1 - } - l.cnt++ - l.sumX += x - l.sumY += y - l.sumXX += x * x - l.sumXY += x * y -} - -func (l *linReg) calc() (b, m float64) { - if l.cnt == 0 { - return 0, 0 - } - cnt := float64(l.cnt) - d := cnt*l.sumXX - l.sumX*l.sumX - if d < 0.001 { - return l.sumY / cnt, 0 - } - m = (cnt*l.sumXY - l.sumX*l.sumY) / d - b = (l.sumY / cnt) - (m * l.sumX / cnt) - return b, m -} - -func (l *linReg) toBytes() []byte { - var arr [40]byte - binary.BigEndian.PutUint64(arr[0:8], math.Float64bits(l.sumX)) - binary.BigEndian.PutUint64(arr[8:16], math.Float64bits(l.sumY)) - binary.BigEndian.PutUint64(arr[16:24], math.Float64bits(l.sumXX)) - binary.BigEndian.PutUint64(arr[24:32], math.Float64bits(l.sumXY)) - binary.BigEndian.PutUint64(arr[32:40], l.cnt) - return arr[:] -} - -func linRegFromBytes(data []byte) *linReg { - if len(data) != 40 { - return nil - } - l := &linReg{} - l.sumX = math.Float64frombits(binary.BigEndian.Uint64(data[0:8])) - l.sumY = math.Float64frombits(binary.BigEndian.Uint64(data[8:16])) - l.sumXX = math.Float64frombits(binary.BigEndian.Uint64(data[16:24])) - l.sumXY = math.Float64frombits(binary.BigEndian.Uint64(data[24:32])) - l.cnt = binary.BigEndian.Uint64(data[32:40]) - return l -} - -type requestCostStats struct { - lock sync.RWMutex - db ethdb.Database - stats map[uint64]*linReg -} - -type requestCostStatsRlp []struct { - MsgCode uint64 - Data []byte -} - -var rcStatsKey = []byte("_requestCostStats") - -func newCostStats(db ethdb.Database) *requestCostStats { - stats := make(map[uint64]*linReg) - for _, code := range reqList { - stats[code] = &linReg{cnt: 100} - } - - if db != nil { - data, err := db.Get(rcStatsKey) - var statsRlp requestCostStatsRlp - if err == nil { - err = rlp.DecodeBytes(data, &statsRlp) - } - if err == nil { - for _, r := range statsRlp { - if stats[r.MsgCode] != nil { - if l := linRegFromBytes(r.Data); l != nil { - stats[r.MsgCode] = l - } - } - } - } - } - - return &requestCostStats{ - db: db, - stats: stats, - } -} - -func (s *requestCostStats) store() { - s.lock.Lock() - defer s.lock.Unlock() - - statsRlp := make(requestCostStatsRlp, len(reqList)) - for i, code := range reqList { - statsRlp[i].MsgCode = code - statsRlp[i].Data = s.stats[code].toBytes() - } - - if data, err := rlp.EncodeToBytes(statsRlp); err == nil { - s.db.Put(rcStatsKey, data) - } -} - -func (s *requestCostStats) getCurrentList() RequestCostList { - s.lock.Lock() - defer s.lock.Unlock() - - list := make(RequestCostList, len(reqList)) - for idx, code := range reqList { - b, m := s.stats[code].calc() - if m < 0 { - b += m - m = 0 - } - if b < 0 { - b = 0 - } - - list[idx].MsgCode = code - list[idx].BaseCost = uint64(b * 2) - list[idx].ReqCost = uint64(m * 2) - } - return list -} - -func (s *requestCostStats) update(msgCode, reqCnt, cost uint64) { - s.lock.Lock() - defer s.lock.Unlock() - - c, ok := s.stats[msgCode] - if !ok || reqCnt == 0 { - return - } - c.add(float64(reqCnt), float64(cost)) -} - func (pm *ProtocolManager) blockLoop() { pm.wg.Add(1) headCh := make(chan core.ChainHeadEvent, 10) @@ -371,12 +293,7 @@ func (pm *ProtocolManager) blockLoop() { switch p.announceType { case announceTypeSimple: - select { - case p.announceChn <- announce: - default: - pm.removePeer(p.id) - } - + p.queueSend(func() { p.SendAnnounce(announce) }) case announceTypeSigned: if !signed { signedAnnounce = announce @@ -384,11 +301,7 @@ func (pm *ProtocolManager) blockLoop() { signed = true } - select { - case p.announceChn <- signedAnnounce: - default: - pm.removePeer(p.id) - } + p.queueSend(func() { p.SendAnnounce(signedAnnounce) }) } } } diff --git a/les/servingqueue.go b/les/servingqueue.go new file mode 100644 index 0000000000..910c85b06e --- /dev/null +++ b/les/servingqueue.go @@ -0,0 +1,191 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package flowcontrol implements a client side flow control mechanism +package les + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/common/prque" +) + +// servingQueue runs serving tasks in a limited number of threads and puts the +// waiting tasks in a priority queue +type servingQueue struct { + lock sync.Mutex + threadCount int // number of currently running threads + stopCount int // number of threads to be stopped after they finish their current task + queue *prque.Prque // priority queue for waiting or suspended tasks + best *servingTask // either best == nil (queue empty) or waitingForTask is empty + waiting []chan *servingTask // threads waiting for a task + suspendBias int64 // priority bias against suspending an already running task +} + +// servingTask represents a request serving task. Tasks can be implemented to +// run in multiple steps, allowing the serving queue to suspend execution between +// steps if higher priority tasks are entered. The creator of the task should +// set the following fields: +// +// - priority: greater value means higher priority; values can wrap around the int64 range +// - run: execute a single step; return true if finished +// - after: executed after run finishes or returns an error, receives the total serving time +type servingTask struct { + servingTime uint64 + done bool + err error + priority int64 + run func() (finished bool, err error) + send func(servingTime uint64) + fail func(err error) +} + +// newServingQueue returns a new servingQueue +func newServingQueue(_suspendBias int64) *servingQueue { + return &servingQueue{ + queue: prque.New(nil), + suspendBias: _suspendBias, + } +} + +// addTask adds a new task, either starting it immediately or queueing it +func (sq *servingQueue) addTask(task *servingTask) { + sq.lock.Lock() + defer sq.lock.Unlock() + + if l := len(sq.waiting); l != 0 { + l-- + sq.waiting[l] <- task + sq.waiting = sq.waiting[:l] + return + } + + if sq.best == nil { + sq.best = task + return + } + if task.priority < sq.best.priority { + sq.queue.Push(sq.best, sq.best.priority) + sq.best = task + return + } + sq.queue.Push(task, task.priority) +} + +// getNewTask selects a new task to be processed. If blocking == true then it waits +// until a runnable task arrives or returns nil if the thread should be stopped. +// if currentTask != nil then it returns immediately and only returns a new task +// if the current one should be suspended. +// Note: either blocking should be false or currentTask should be nil. +func (sq *servingQueue) getNewTask(currentTask *servingTask, blocking bool) *servingTask { + sq.lock.Lock() + if sq.stopCount != 0 { + } + if sq.stopCount == 0 { + if sq.best != nil && (currentTask == nil || sq.best.priority <= currentTask.priority-sq.suspendBias) { + best := sq.best + if sq.queue.Size() == 0 { + sq.best = nil + } else { + sq.best, _ = sq.queue.PopItem().(*servingTask) + } + sq.lock.Unlock() + return best + } + if blocking { + ch := make(chan *servingTask) + sq.waiting = append(sq.waiting, ch) + sq.lock.Unlock() + return <-ch + } + } else { + sq.stopCount-- + sq.threadCount-- + } + sq.lock.Unlock() + return nil +} + +// setThreads sets the processing thread count, suspending tasks as soon as +// possible if necessary. +func (sq *servingQueue) setThreads(threadCount int) { + sq.lock.Lock() + defer sq.lock.Unlock() + + diff := threadCount - sq.threadCount + sq.stopCount + if diff > 0 { + // start more threads + if sq.stopCount >= diff { + sq.stopCount -= diff + } else { + diff -= sq.stopCount + sq.stopCount = 0 + sq.threadCount += diff + for ; diff > 0; diff-- { + go sq.servingThread() + } + } + } + if diff < 0 { + // stop some threads + lw := len(sq.waiting) + sq.stopCount -= diff + for diff < 0 && lw > 0 { + diff++ + lw-- + sq.waiting[lw] <- nil + sq.stopCount-- + sq.threadCount-- + } + sq.waiting = sq.waiting[:lw] + } +} + +// stop stops task processing as soon as possible +func (sq *servingQueue) stop() { + sq.setThreads(0) +} + +// servingThread implements a single serving thread +func (sq *servingQueue) servingThread() { + for { + task := sq.getNewTask(nil, true) + if task == nil { + return + } + task.servingTime -= uint64(mclock.Now()) + for { + task.done, task.err = task.run() + if task.done || task.err != nil { + task.servingTime += uint64(mclock.Now()) + if task.err == nil { + task.send(task.servingTime) + } else { + task.fail(task.err) + } + break + } + if newTask := sq.getNewTask(task, false); newTask != nil { + now := uint64(mclock.Now()) + task.servingTime += now + sq.addTask(task) + task = newTask + task.servingTime -= now + } + } + } +} diff --git a/les/txrelay.go b/les/txrelay.go index e5e0bcab0c..a790bbec9c 100644 --- a/les/txrelay.go +++ b/les/txrelay.go @@ -128,7 +128,7 @@ func (self *LesTxRelay) send(txs types.Transactions, count int) { request: func(dp distPeer) func() { peer := dp.(*peer) cost := peer.GetTxRelayCost(len(ll), len(enc)) - peer.fcServer.QueueRequest(reqID, cost) + peer.fcServer.QueuedRequest(reqID, cost) return func() { peer.SendTxs(reqID, cost, enc) } }, } diff --git a/node/config.go b/node/config.go index 2f871e478b..fd0d330c36 100644 --- a/node/config.go +++ b/node/config.go @@ -337,7 +337,13 @@ func (c *Config) instanceDir() string { if c.DataDir == "" { return "" } - return filepath.Join(c.DataDir, c.name()) + name := c.name() + if name == "p2p-node" { + // use original data dir with network simulator in order to allow using + // an existing database for a simulated node instead of a temporary one + name = "geth" + } + return filepath.Join(c.DataDir, name) } // NodeKey retrieves the currently configured private key of the node, checking diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 9b588db1be..d61d165f41 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -97,7 +97,11 @@ func (e *ExecAdapter) NewNode(config *NodeConfig) (Node, error) { Stack: node.DefaultConfig, Node: config, } - conf.Stack.DataDir = filepath.Join(dir, "data") + if config.DataDir != "" { + conf.Stack.DataDir = config.DataDir + } else { + conf.Stack.DataDir = filepath.Join(dir, "data") + } conf.Stack.WSHost = "127.0.0.1" conf.Stack.WSPort = 0 conf.Stack.WSOrigins = []string{"*"} @@ -177,7 +181,7 @@ func (n *ExecNode) Start(snapshots map[string][]byte) (err error) { } // start the one-shot server that waits for startup information - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) defer cancel() statusURL, statusC := n.waitForStartupJSON(ctx) diff --git a/p2p/simulations/adapters/types.go b/p2p/simulations/adapters/types.go index 6681726e40..31856b76dd 100644 --- a/p2p/simulations/adapters/types.go +++ b/p2p/simulations/adapters/types.go @@ -90,6 +90,9 @@ type NodeConfig struct { // Name is a human friendly name for the node like "node01" Name string + // Use an existing database instead of a temporary one if non-empty + DataDir string + // Services are the names of the services which should be run when // starting the node (for SimNodes it should be the names of services // contained in SimAdapter.services, for other nodes it should be