diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index bd342a8cb9..fe7dea4da1 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -428,10 +428,23 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb } return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil } + func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)), nil } +func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { + receipts := core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)) + if receipts == nil { + return nil, nil + } + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs, nil +} + func (fb *filterBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { return event.NewSubscription(func(quit <-chan struct{}) error { <-quit diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 99527f9d1e..095668c868 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -686,8 +686,6 @@ func authTwitter(url string) (string, string, common.Address, error) { if len(parts) < 4 || parts[len(parts)-2] != "status" { return "", "", common.Address{}, errors.New("Invalid Twitter status URL") } - username := parts[len(parts)-3] - // Twitter's API isn't really friendly with direct links. Still, we don't // want to do ask read permissions from users, so just load the public posts and // scrape it for the Ethereum address and profile URL. @@ -697,6 +695,13 @@ func authTwitter(url string) (string, string, common.Address, error) { } defer res.Body.Close() + // Resolve the username from the final redirect, no intermediate junk + parts = strings.Split(res.Request.URL.String(), "/") + if len(parts) < 4 || parts[len(parts)-2] != "status" { + return "", "", common.Address{}, errors.New("Invalid Twitter status URL") + } + username := parts[len(parts)-3] + body, err := ioutil.ReadAll(res.Body) if err != nil { return "", "", common.Address{}, err diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 7eca4d59f6..2500a969cf 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -43,7 +43,7 @@ var ( Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. -See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.`, +See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console.`, } attachCommand = cli.Command{ @@ -56,7 +56,7 @@ See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console.`, Description: ` The Geth console is an interactive shell for the JavaScript runtime environment which exposes a node admin interface as well as the Ðapp JavaScript API. -See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console. +See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console. This command allows to open a console on a running geth node.`, } @@ -69,7 +69,7 @@ This command allows to open a console on a running geth node.`, Category: "CONSOLE COMMANDS", Description: ` The JavaScript VM exposes a node admin interface as well as the Ðapp -JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Console`, +JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console`, } ) diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index 1cb2d45491..3832b247f8 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -631,6 +631,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da "Tangerine": conf.Genesis.Config.EIP150Block, "Spurious": conf.Genesis.Config.EIP155Block, "Byzantium": conf.Genesis.Config.ByzantiumBlock, + "Constantinople": conf.Genesis.Config.ConstantinopleBlock, }) files[filepath.Join(workdir, "index.html")] = indexfile.Bytes() diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index c2209a3204..2bafe97228 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -23,6 +23,7 @@ import ( "os" "reflect" "strconv" + "strings" "unicode" cli "gopkg.in/urfave/cli.v1" @@ -102,10 +103,15 @@ func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) { config = bzzapi.NewConfig() //first load settings from config file (if provided) config, err = configFileOverride(config, ctx) + if err != nil { + return nil, err + } //override settings provided by environment variables config = envVarsOverride(config) //override settings provided by command line config = cmdLineOverride(config, ctx) + //validate configuration parameters + err = validateConfig(config) return } @@ -199,12 +205,16 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) } - //EnsApi can be set to "", so can't check for empty string, as it is allowed! if ctx.GlobalIsSet(EnsAPIFlag.Name) { - currentConfig.EnsApi = ctx.GlobalString(EnsAPIFlag.Name) + ensAPIs := ctx.GlobalStringSlice(EnsAPIFlag.Name) + // preserve backward compatibility to disable ENS with --ens-api="" + if len(ensAPIs) == 1 && ensAPIs[0] == "" { + ensAPIs = nil + } + currentConfig.EnsAPIs = ensAPIs } - if ensaddr := ctx.GlobalString(EnsAddrFlag.Name); ensaddr != "" { + if ensaddr := ctx.GlobalString(DeprecatedEnsAddrFlag.Name); ensaddr != "" { currentConfig.EnsRoot = common.HexToAddress(ensaddr) } @@ -291,9 +301,8 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) } - //EnsApi can be set to "", so can't check for empty string, as it is allowed - if ensapi, exists := os.LookupEnv(SWARM_ENV_ENS_API); exists { - currentConfig.EnsApi = ensapi + if ensapi := os.Getenv(SWARM_ENV_ENS_API); ensapi != "" { + currentConfig.EnsAPIs = strings.Split(ensapi, ",") } if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" { @@ -340,6 +349,43 @@ func checkDeprecated(ctx *cli.Context) { if ctx.GlobalString(DeprecatedEthAPIFlag.Name) != "" { utils.Fatalf("--ethapi is no longer a valid command line flag, please use --ens-api and/or --swap-api.") } + // warn if --ens-api flag is set + if ctx.GlobalString(DeprecatedEnsAddrFlag.Name) != "" { + log.Warn("--ens-addr is no longer a valid command line flag, please use --ens-api to specify contract address.") + } +} + +//validate configuration parameters +func validateConfig(cfg *bzzapi.Config) (err error) { + for _, ensAPI := range cfg.EnsAPIs { + if ensAPI != "" { + if err := validateEnsAPIs(ensAPI); err != nil { + return fmt.Errorf("invalid format [tld:][contract-addr@]url for ENS API endpoint configuration %q: %v", ensAPI, err) + } + } + } + return nil +} + +//validate EnsAPIs configuration parameter +func validateEnsAPIs(s string) (err error) { + // missing contract address + if strings.HasPrefix(s, "@") { + return errors.New("missing contract address") + } + // missing url + if strings.HasSuffix(s, "@") { + return errors.New("missing url") + } + // missing tld + if strings.HasPrefix(s, ":") { + return errors.New("missing tld") + } + // missing url + if strings.HasSuffix(s, ":") { + return errors.New("missing url") + } + return nil } //print a Config as string diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go index 81d1df585d..57396d1408 100644 --- a/cmd/swarm/config_test.go +++ b/cmd/swarm/config_test.go @@ -478,3 +478,98 @@ func TestConfigCmdLineOverridesFile(t *testing.T) { node.Shutdown() } + +func TestValidateConfig(t *testing.T) { + for _, c := range []struct { + cfg *api.Config + err string + }{ + { + cfg: &api.Config{EnsAPIs: []string{ + "/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "http://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "ws://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "test:/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "test:ws://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "test:314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "eth:314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "eth:314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:12344", + }}, + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "eth:", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"eth:\": missing url", + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "314159265dD8dbb310642f98f50C066173C1259b@", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"314159265dD8dbb310642f98f50C066173C1259b@\": missing url", + }, + { + cfg: &api.Config{EnsAPIs: []string{ + ":314159265dD8dbb310642f98f50C066173C1259", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \":314159265dD8dbb310642f98f50C066173C1259\": missing tld", + }, + { + cfg: &api.Config{EnsAPIs: []string{ + "@/data/testnet/geth.ipc", + }}, + err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"@/data/testnet/geth.ipc\": missing contract address", + }, + } { + err := validateConfig(c.cfg) + if c.err != "" && err.Error() != c.err { + t.Errorf("expected error %q, got %q", c.err, err) + } + if c.err == "" && err != nil { + t.Errorf("unexpected error %q", err) + } + } +} diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 3ae45d4c2f..6c5693d4d2 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -17,11 +17,9 @@ package main import ( - "context" "crypto/ecdsa" "fmt" "io/ioutil" - "math/big" "os" "os/signal" "runtime" @@ -29,14 +27,12 @@ import ( "strconv" "strings" "syscall" - "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/console" - "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/internal/debug" @@ -45,9 +41,9 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" + swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics" "gopkg.in/urfave/cli.v1" ) @@ -110,16 +106,11 @@ var ( Usage: "Swarm Syncing enabled (default true)", EnvVar: SWARM_ENV_SYNC_ENABLE, } - EnsAPIFlag = cli.StringFlag{ + EnsAPIFlag = cli.StringSliceFlag{ Name: "ens-api", - Usage: "URL of the Ethereum API provider to use for ENS record lookups", + Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", EnvVar: SWARM_ENV_ENS_API, } - EnsAddrFlag = cli.StringFlag{ - Name: "ens-addr", - Usage: "ENS contract address (default is detected as testnet or mainnet using --ens-api)", - EnvVar: SWARM_ENV_ENS_ADDR, - } SwarmApiFlag = cli.StringFlag{ Name: "bzzapi", Usage: "Swarm HTTP endpoint", @@ -180,6 +171,10 @@ var ( Name: "ethapi", Usage: "DEPRECATED: please use --ens-api and --swap-api", } + DeprecatedEnsAddrFlag = cli.StringFlag{ + Name: "ens-addr", + Usage: "DEPRECATED: ENS contract address, please use --ens-api with contract address according to its format", + } ) //declare a few constant error messages, useful for later error check comparisons in test @@ -367,7 +362,6 @@ DEPRECATED: use 'swarm db clean'. // bzzd-specific flags CorsStringFlag, EnsAPIFlag, - EnsAddrFlag, SwarmTomlConfigPathFlag, SwarmConfigPathFlag, SwarmSwapEnabledFlag, @@ -394,6 +388,7 @@ DEPRECATED: use 'swarm db clean'. SwarmStoreRadius, //deprecated flags DeprecatedEthAPIFlag, + DeprecatedEnsAddrFlag, } rpcFlags := []cli.Flag{ utils.WSEnabledFlag, @@ -404,9 +399,14 @@ DEPRECATED: use 'swarm db clean'. } app.Flags = append(app.Flags, rpcFlags...) app.Flags = append(app.Flags, debug.Flags...) + app.Flags = append(app.Flags, swarmmetrics.Flags...) app.Before = func(ctx *cli.Context) error { runtime.GOMAXPROCS(runtime.NumCPU()) - return debug.Setup(ctx) + if err := debug.Setup(ctx); err != nil { + return err + } + swarmmetrics.Setup(ctx) + return nil } app.After = func(ctx *cli.Context) error { debug.Exit() @@ -493,38 +493,6 @@ func bzzd(ctx *cli.Context) error { return nil } -// detectEnsAddr determines the ENS contract address by getting both the -// version and genesis hash using the client and matching them to either -// mainnet or testnet addresses -func detectEnsAddr(client *rpc.Client) (common.Address, error) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - var version string - if err := client.CallContext(ctx, &version, "net_version"); err != nil { - return common.Address{}, err - } - - block, err := ethclient.NewClient(client).BlockByNumber(ctx, big.NewInt(0)) - if err != nil { - return common.Address{}, err - } - - switch { - - case version == "1" && block.Hash() == params.MainnetGenesisHash: - log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress) - return ens.MainNetAddress, nil - - case version == "3" && block.Hash() == params.TestnetGenesisHash: - log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress) - return ens.TestNetAddress, nil - - default: - return common.Address{}, fmt.Errorf("unknown version and genesis hash: %s %s", version, block.Hash()) - } -} - func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.Node) { //define the swarm service boot function @@ -539,28 +507,8 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node. } } - var ensClient *ethclient.Client - if bzzconfig.EnsApi != "" { - log.Info("connecting to ENS API", "url", bzzconfig.EnsApi) - client, err := rpc.Dial(bzzconfig.EnsApi) - if err != nil { - return nil, fmt.Errorf("error connecting to ENS API %s: %s", bzzconfig.EnsApi, err) - } - ensClient = ethclient.NewClient(client) - - //no ENS root address set yet - if bzzconfig.EnsRoot == (common.Address{}) { - ensAddr, err := detectEnsAddr(client) - if err == nil { - bzzconfig.EnsRoot = ensAddr - } else { - log.Warn(fmt.Sprintf("could not determine ENS contract address, using default %s", bzzconfig.EnsRoot), "err", err) - } - } - } - // In production, mockStore must be always nil. - return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, nil) + return swarm.NewSwarm(ctx, swapClient, bzzconfig, nil) } //register within the ethereum node if err := stack.Register(boot); err != nil { diff --git a/common/big.go b/common/big.go index b552608bc7..65d4377bf7 100644 --- a/common/big.go +++ b/common/big.go @@ -25,6 +25,6 @@ var ( Big3 = big.NewInt(3) Big0 = big.NewInt(0) Big32 = big.NewInt(32) - Big256 = big.NewInt(0xff) + Big256 = big.NewInt(256) Big257 = big.NewInt(257) ) diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 91e20112ae..1b3dcee302 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -35,9 +35,9 @@ import ( mmap "github.com/edsrzf/mmap-go" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rpc" "github.com/hashicorp/golang-lru/simplelru" - metrics "github.com/rcrowley/go-metrics" ) var ErrInvalidDumpMagic = errors.New("invalid dump magic") diff --git a/core/blockchain.go b/core/blockchain.go index 4ae0e4f4ec..6006e66740 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -46,7 +46,7 @@ import ( ) var ( - blockInsertTimer = metrics.NewTimer("chain/inserts") + blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) ErrNoGenesis = errors.New("Genesis not found in chain") ) @@ -107,8 +107,8 @@ type BlockChain struct { procmu sync.RWMutex // block processor lock checkpoint int // checkpoint counts towards the new checkpoint - currentBlock *types.Block // Current head of the block chain - currentFastBlock *types.Block // Current head of the fast-sync chain (may be above the block chain!) + currentBlock atomic.Value // Current head of the block chain + currentFastBlock atomic.Value // Current head of the fast-sync chain (may be above the block chain!) stateCache state.Database // State database to reuse between imports (contains state cache) bodyCache *lru.Cache // Cache for the most recent block bodies @@ -224,10 +224,10 @@ func (bc *BlockChain) loadLastState() error { } } // Everything seems to be fine, set as the head block - bc.currentBlock = currentBlock + bc.currentBlock.Store(currentBlock) // Restore the last known head header - currentHeader := bc.currentBlock.Header() + currentHeader := currentBlock.Header() if head := GetHeadHeaderHash(bc.db); head != (common.Hash{}) { if header := bc.GetHeaderByHash(head); header != nil { currentHeader = header @@ -236,21 +236,23 @@ func (bc *BlockChain) loadLastState() error { bc.hc.SetCurrentHeader(currentHeader) // Restore the last known head fast block - bc.currentFastBlock = bc.currentBlock + bc.currentFastBlock.Store(currentBlock) if head := GetHeadFastBlockHash(bc.db); head != (common.Hash{}) { if block := bc.GetBlockByHash(head); block != nil { - bc.currentFastBlock = block + bc.currentFastBlock.Store(block) } } // Issue a status log for the user + currentFastBlock := bc.CurrentFastBlock() + headerTd := bc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()) - blockTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()) - fastTd := bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()) + blockTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) + fastTd := bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()) log.Info("Loaded most recent local header", "number", currentHeader.Number, "hash", currentHeader.Hash(), "td", headerTd) - log.Info("Loaded most recent local full block", "number", bc.currentBlock.Number(), "hash", bc.currentBlock.Hash(), "td", blockTd) - log.Info("Loaded most recent local fast block", "number", bc.currentFastBlock.Number(), "hash", bc.currentFastBlock.Hash(), "td", fastTd) + log.Info("Loaded most recent local full block", "number", currentBlock.Number(), "hash", currentBlock.Hash(), "td", blockTd) + log.Info("Loaded most recent local fast block", "number", currentFastBlock.Number(), "hash", currentFastBlock.Hash(), "td", fastTd) return nil } @@ -279,30 +281,32 @@ func (bc *BlockChain) SetHead(head uint64) error { bc.futureBlocks.Purge() // Rewind the block chain, ensuring we don't end up with a stateless head block - if bc.currentBlock != nil && currentHeader.Number.Uint64() < bc.currentBlock.NumberU64() { - bc.currentBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) + if currentBlock := bc.CurrentBlock(); currentBlock != nil && currentHeader.Number.Uint64() < currentBlock.NumberU64() { + bc.currentBlock.Store(bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())) } - if bc.currentBlock != nil { - if _, err := state.New(bc.currentBlock.Root(), bc.stateCache); err != nil { + if currentBlock := bc.CurrentBlock(); currentBlock != nil { + if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil { // Rewound state missing, rolled back to before pivot, reset to genesis - bc.currentBlock = nil + bc.currentBlock.Store(bc.genesisBlock) } } // Rewind the fast block in a simpleton way to the target head - if bc.currentFastBlock != nil && currentHeader.Number.Uint64() < bc.currentFastBlock.NumberU64() { - bc.currentFastBlock = bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64()) + if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock != nil && currentHeader.Number.Uint64() < currentFastBlock.NumberU64() { + bc.currentFastBlock.Store(bc.GetBlock(currentHeader.Hash(), currentHeader.Number.Uint64())) } // If either blocks reached nil, reset to the genesis state - if bc.currentBlock == nil { - bc.currentBlock = bc.genesisBlock + if currentBlock := bc.CurrentBlock(); currentBlock == nil { + bc.currentBlock.Store(bc.genesisBlock) } - if bc.currentFastBlock == nil { - bc.currentFastBlock = bc.genesisBlock + if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock == nil { + bc.currentFastBlock.Store(bc.genesisBlock) } - if err := WriteHeadBlockHash(bc.db, bc.currentBlock.Hash()); err != nil { + currentBlock := bc.CurrentBlock() + currentFastBlock := bc.CurrentFastBlock() + if err := WriteHeadBlockHash(bc.db, currentBlock.Hash()); err != nil { log.Crit("Failed to reset head full block", "err", err) } - if err := WriteHeadFastBlockHash(bc.db, bc.currentFastBlock.Hash()); err != nil { + if err := WriteHeadFastBlockHash(bc.db, currentFastBlock.Hash()); err != nil { log.Crit("Failed to reset head fast block", "err", err) } return bc.loadLastState() @@ -321,7 +325,7 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { } // If all checks out, manually set the head block bc.mu.Lock() - bc.currentBlock = block + bc.currentBlock.Store(block) bc.mu.Unlock() log.Info("Committed new head block", "number", block.Number(), "hash", hash) @@ -330,28 +334,19 @@ func (bc *BlockChain) FastSyncCommitHead(hash common.Hash) error { // GasLimit returns the gas limit of the current HEAD block. func (bc *BlockChain) GasLimit() uint64 { - bc.mu.RLock() - defer bc.mu.RUnlock() - - return bc.currentBlock.GasLimit() + return bc.CurrentBlock().GasLimit() } // CurrentBlock retrieves the current head block of the canonical chain. The // block is retrieved from the blockchain's internal cache. func (bc *BlockChain) CurrentBlock() *types.Block { - bc.mu.RLock() - defer bc.mu.RUnlock() - - return bc.currentBlock + return bc.currentBlock.Load().(*types.Block) } // CurrentFastBlock retrieves the current fast-sync head block of the canonical // chain. The block is retrieved from the blockchain's internal cache. func (bc *BlockChain) CurrentFastBlock() *types.Block { - bc.mu.RLock() - defer bc.mu.RUnlock() - - return bc.currentFastBlock + return bc.currentFastBlock.Load().(*types.Block) } // SetProcessor sets the processor required for making state modifications. @@ -416,10 +411,10 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { } bc.genesisBlock = genesis bc.insert(bc.genesisBlock) - bc.currentBlock = bc.genesisBlock + bc.currentBlock.Store(bc.genesisBlock) bc.hc.SetGenesis(bc.genesisBlock.Header()) bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) - bc.currentFastBlock = bc.genesisBlock + bc.currentFastBlock.Store(bc.genesisBlock) return nil } @@ -444,7 +439,7 @@ func (bc *BlockChain) repair(head **types.Block) error { // Export writes the active chain to the given writer. func (bc *BlockChain) Export(w io.Writer) error { - return bc.ExportN(w, uint64(0), bc.currentBlock.NumberU64()) + return bc.ExportN(w, uint64(0), bc.CurrentBlock().NumberU64()) } // ExportN writes a subset of the active chain to the given writer. @@ -488,7 +483,7 @@ func (bc *BlockChain) insert(block *types.Block) { if err := WriteHeadBlockHash(bc.db, block.Hash()); err != nil { log.Crit("Failed to insert head block hash", "err", err) } - bc.currentBlock = block + bc.currentBlock.Store(block) // If the block is better than our head or is on a different chain, force update heads if updateHeads { @@ -497,7 +492,7 @@ func (bc *BlockChain) insert(block *types.Block) { if err := WriteHeadFastBlockHash(bc.db, block.Hash()); err != nil { log.Crit("Failed to insert head fast block hash", "err", err) } - bc.currentFastBlock = block + bc.currentFastBlock.Store(block) } } @@ -648,22 +643,21 @@ func (bc *BlockChain) Stop() { bc.wg.Wait() // Ensure the state of a recent block is also stored to disk before exiting. - // It is fine if this state does not exist (fast start/stop cycle), but it is - // advisable to leave an N block gap from the head so 1) a restart loads up - // the last N blocks as sync assistance to remote nodes; 2) a restart during - // a (small) reorg doesn't require deep reprocesses; 3) chain "repair" from - // missing states are constantly tested. - // - // This may be tuned a bit on mainnet if its too annoying to reprocess the last - // N blocks. + // We're writing three different states to catch different restart scenarios: + // - HEAD: So we don't need to reprocess any blocks in the general case + // - HEAD-1: So we don't do large reorgs if our HEAD becomes an uncle + // - HEAD-127: So we have a hard limit on the number of blocks reexecuted if !bc.cacheConfig.Disabled { triedb := bc.stateCache.TrieDB() - if number := bc.CurrentBlock().NumberU64(); number >= triesInMemory { - recent := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - triesInMemory + 1) - log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) - if err := triedb.Commit(recent.Root(), true); err != nil { - log.Error("Failed to commit recent state trie", "err", err) + for _, offset := range []uint64{0, 1, triesInMemory - 1} { + if number := bc.CurrentBlock().NumberU64(); number > offset { + recent := bc.GetBlockByNumber(number - offset) + + log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root()) + if err := triedb.Commit(recent.Root(), true); err != nil { + log.Error("Failed to commit recent state trie", "err", err) + } } } for !bc.triegc.Empty() { @@ -715,13 +709,15 @@ func (bc *BlockChain) Rollback(chain []common.Hash) { if currentHeader.Hash() == hash { bc.hc.SetCurrentHeader(bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)) } - if bc.currentFastBlock.Hash() == hash { - bc.currentFastBlock = bc.GetBlock(bc.currentFastBlock.ParentHash(), bc.currentFastBlock.NumberU64()-1) - WriteHeadFastBlockHash(bc.db, bc.currentFastBlock.Hash()) + if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock.Hash() == hash { + newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1) + bc.currentFastBlock.Store(newFastBlock) + WriteHeadFastBlockHash(bc.db, newFastBlock.Hash()) } - if bc.currentBlock.Hash() == hash { - bc.currentBlock = bc.GetBlock(bc.currentBlock.ParentHash(), bc.currentBlock.NumberU64()-1) - WriteHeadBlockHash(bc.db, bc.currentBlock.Hash()) + if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash { + newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1) + bc.currentBlock.Store(newBlock) + WriteHeadBlockHash(bc.db, newBlock.Hash()) } } } @@ -830,11 +826,12 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ bc.mu.Lock() head := blockChain[len(blockChain)-1] if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case - if bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()).Cmp(td) < 0 { + currentFastBlock := bc.CurrentFastBlock() + if bc.GetTd(currentFastBlock.Hash(), currentFastBlock.NumberU64()).Cmp(td) < 0 { if err := WriteHeadFastBlockHash(bc.db, head.Hash()); err != nil { log.Crit("Failed to update head fast block hash", "err", err) } - bc.currentFastBlock = head + bc.currentFastBlock.Store(head) } } bc.mu.Unlock() @@ -881,7 +878,8 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. bc.mu.Lock() defer bc.mu.Unlock() - localTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()) + currentBlock := bc.CurrentBlock() + localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) externTd := new(big.Int).Add(block.Difficulty(), ptd) // Irrelevant of the canonical status, write the block itself to the database @@ -956,14 +954,15 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. // Second clause in the if statement reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf reorg := externTd.Cmp(localTd) > 0 + currentBlock = bc.CurrentBlock() if !reorg && externTd.Cmp(localTd) == 0 { // Split same-difficulty blocks by number, then at random - reorg = block.NumberU64() < bc.currentBlock.NumberU64() || (block.NumberU64() == bc.currentBlock.NumberU64() && mrand.Float64() < 0.5) + reorg = block.NumberU64() < currentBlock.NumberU64() || (block.NumberU64() == currentBlock.NumberU64() && mrand.Float64() < 0.5) } if reorg { // Reorganise the chain if the parent is not the head block - if block.ParentHash() != bc.currentBlock.Hash() { - if err := bc.reorg(bc.currentBlock, block); err != nil { + if block.ParentHash() != currentBlock.Hash() { + if err := bc.reorg(currentBlock, block); err != nil { return NonStatTy, err } } @@ -1092,7 +1091,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*ty case err == consensus.ErrPrunedAncestor: // Block competing with the canonical chain, store in the db, but don't process // until the competitor TD goes above the canonical TD - localTd := bc.GetTd(bc.currentBlock.Hash(), bc.currentBlock.NumberU64()) + currentBlock := bc.CurrentBlock() + localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) externTd := new(big.Int).Add(bc.GetTd(block.ParentHash(), block.NumberU64()-1), block.Difficulty()) if localTd.Cmp(externTd) > 0 { if err = bc.WriteBlockWithoutState(block, externTd); err != nil { @@ -1481,9 +1481,6 @@ func (bc *BlockChain) writeHeader(header *types.Header) error { // CurrentHeader retrieves the current head header of the canonical chain. The // header is retrieved from the HeaderChain's internal cache. func (bc *BlockChain) CurrentHeader() *types.Header { - bc.mu.RLock() - defer bc.mu.RUnlock() - return bc.hc.CurrentHeader() } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 635379161c..748cdc5c71 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -34,26 +34,6 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// newTestBlockChain creates a blockchain without validation. -func newTestBlockChain(fake bool) *BlockChain { - db, _ := ethdb.NewMemDatabase() - gspec := &Genesis{ - Config: params.TestChainConfig, - Difficulty: big.NewInt(1), - } - gspec.MustCommit(db) - engine := ethash.NewFullFaker() - if !fake { - engine = ethash.NewTester() - } - blockchain, err := NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) - if err != nil { - panic(err) - } - blockchain.SetValidator(bproc{}) - return blockchain -} - // Test fork of length N starting from block i func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int)) { // Copy old chain up to #i into a new db @@ -183,13 +163,18 @@ func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t * } func TestLastBlock(t *testing.T) { - bchain := newTestBlockChain(false) - defer bchain.Stop() + _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() - block := makeBlockChain(bchain.CurrentBlock(), 1, ethash.NewFaker(), bchain.db, 0)[0] - bchain.insert(block) - if block.Hash() != GetHeadBlockHash(bchain.db) { - t.Errorf("Write/Get HeadBlockHash failed") + blocks := makeBlockChain(blockchain.CurrentBlock(), 1, ethash.NewFullFaker(), blockchain.db, 0) + if _, err := blockchain.InsertChain(blocks); err != nil { + t.Fatalf("Failed to insert block: %v", err) + } + if blocks[len(blocks)-1].Hash() != GetHeadBlockHash(blockchain.db) { + t.Fatalf("Write/Get HeadBlockHash failed") } } @@ -337,55 +322,13 @@ func testBrokenChain(t *testing.T, full bool) { } } -type bproc struct{} - -func (bproc) ValidateBody(*types.Block) error { return nil } -func (bproc) ValidateState(block, parent *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64) error { - return nil -} -func (bproc) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (types.Receipts, []*types.Log, uint64, error) { - return nil, nil, 0, nil -} - -func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Header { - blocks := makeBlockChainWithDiff(genesis, d, seed) - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - return headers -} - -func makeBlockChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block { - var chain []*types.Block - for i, difficulty := range d { - header := &types.Header{ - Coinbase: common.Address{seed}, - Number: big.NewInt(int64(i + 1)), - Difficulty: big.NewInt(int64(difficulty)), - UncleHash: types.EmptyUncleHash, - TxHash: types.EmptyRootHash, - ReceiptHash: types.EmptyRootHash, - Time: big.NewInt(int64(i) + 1), - } - if i == 0 { - header.ParentHash = genesis.Hash() - } else { - header.ParentHash = chain[i-1].Hash() - } - block := types.NewBlockWithHeader(header) - chain = append(chain, block) - } - return chain -} - // Tests that reorganising a long difficult chain after a short easy one // overwrites the canonical numbers and links in the database. func TestReorgLongHeaders(t *testing.T) { testReorgLong(t, false) } func TestReorgLongBlocks(t *testing.T) { testReorgLong(t, true) } func testReorgLong(t *testing.T, full bool) { - testReorg(t, []int{1, 2, 4}, []int{1, 2, 3, 4}, 10, full) + testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280, full) } // Tests that reorganising a short difficult chain after a long easy one @@ -394,45 +337,82 @@ func TestReorgShortHeaders(t *testing.T) { testReorgShort(t, false) } func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true) } func testReorgShort(t *testing.T, full bool) { - testReorg(t, []int{1, 2, 3, 4}, []int{1, 10}, 11, full) + // Create a long easy chain vs. a short heavy one. Due to difficulty adjustment + // we need a fairly long chain of blocks with different difficulties for a short + // one to become heavyer than a long one. The 96 is an empirical value. + easy := make([]int64, 96) + for i := 0; i < len(easy); i++ { + easy[i] = 60 + } + diff := make([]int64, len(easy)-1) + for i := 0; i < len(diff); i++ { + diff[i] = -9 + } + testReorg(t, easy, diff, 12615120, full) } -func testReorg(t *testing.T, first, second []int, td int64, full bool) { - bc := newTestBlockChain(true) - defer bc.Stop() +func testReorg(t *testing.T, first, second []int64, td int64, full bool) { + // Create a pristine chain and database + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() // Insert an easy and a difficult chain afterwards + easyBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(first), func(i int, b *BlockGen) { + b.OffsetTime(first[i]) + }) + diffBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(second), func(i int, b *BlockGen) { + b.OffsetTime(second[i]) + }) if full { - bc.InsertChain(makeBlockChainWithDiff(bc.genesisBlock, first, 11)) - bc.InsertChain(makeBlockChainWithDiff(bc.genesisBlock, second, 22)) + if _, err := blockchain.InsertChain(easyBlocks); err != nil { + t.Fatalf("failed to insert easy chain: %v", err) + } + if _, err := blockchain.InsertChain(diffBlocks); err != nil { + t.Fatalf("failed to insert difficult chain: %v", err) + } } else { - bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, first, 11), 1) - bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, second, 22), 1) + easyHeaders := make([]*types.Header, len(easyBlocks)) + for i, block := range easyBlocks { + easyHeaders[i] = block.Header() + } + diffHeaders := make([]*types.Header, len(diffBlocks)) + for i, block := range diffBlocks { + diffHeaders[i] = block.Header() + } + if _, err := blockchain.InsertHeaderChain(easyHeaders, 1); err != nil { + t.Fatalf("failed to insert easy chain: %v", err) + } + if _, err := blockchain.InsertHeaderChain(diffHeaders, 1); err != nil { + t.Fatalf("failed to insert difficult chain: %v", err) + } } // Check that the chain is valid number and link wise if full { - prev := bc.CurrentBlock() - for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) { + prev := blockchain.CurrentBlock() + for block := blockchain.GetBlockByNumber(blockchain.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, blockchain.GetBlockByNumber(block.NumberU64()-1) { if prev.ParentHash() != block.Hash() { t.Errorf("parent block hash mismatch: have %x, want %x", prev.ParentHash(), block.Hash()) } } } else { - prev := bc.CurrentHeader() - for header := bc.GetHeaderByNumber(bc.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, bc.GetHeaderByNumber(header.Number.Uint64()-1) { + prev := blockchain.CurrentHeader() + for header := blockchain.GetHeaderByNumber(blockchain.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, blockchain.GetHeaderByNumber(header.Number.Uint64()-1) { if prev.ParentHash != header.Hash() { t.Errorf("parent header hash mismatch: have %x, want %x", prev.ParentHash, header.Hash()) } } } // Make sure the chain total difficulty is the correct one - want := new(big.Int).Add(bc.genesisBlock.Difficulty(), big.NewInt(td)) + want := new(big.Int).Add(blockchain.genesisBlock.Difficulty(), big.NewInt(td)) if full { - if have := bc.GetTdByHash(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 { + if have := blockchain.GetTdByHash(blockchain.CurrentBlock().Hash()); have.Cmp(want) != 0 { t.Errorf("total difficulty mismatch: have %v, want %v", have, want) } } else { - if have := bc.GetTdByHash(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 { + if have := blockchain.GetTdByHash(blockchain.CurrentHeader().Hash()); have.Cmp(want) != 0 { t.Errorf("total difficulty mismatch: have %v, want %v", have, want) } } @@ -443,19 +423,28 @@ func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false) } func TestBadBlockHashes(t *testing.T) { testBadHashes(t, true) } func testBadHashes(t *testing.T, full bool) { - bc := newTestBlockChain(true) - defer bc.Stop() + // Create a pristine chain and database + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() // Create a chain, ban a hash and try to import - var err error if full { - blocks := makeBlockChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10) + blocks := makeBlockChain(blockchain.CurrentBlock(), 3, ethash.NewFaker(), db, 10) + BadHashes[blocks[2].Header().Hash()] = true - _, err = bc.InsertChain(blocks) + defer func() { delete(BadHashes, blocks[2].Header().Hash()) }() + + _, err = blockchain.InsertChain(blocks) } else { - headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10) + headers := makeHeaderChain(blockchain.CurrentHeader(), 3, ethash.NewFaker(), db, 10) + BadHashes[headers[2].Hash()] = true - _, err = bc.InsertHeaderChain(headers, 1) + defer func() { delete(BadHashes, headers[2].Hash()) }() + + _, err = blockchain.InsertHeaderChain(headers, 1) } if err != ErrBlacklistedHash { t.Errorf("error mismatch: have: %v, want: %v", err, ErrBlacklistedHash) @@ -468,40 +457,41 @@ func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false) } func TestReorgBadBlockHashes(t *testing.T) { testReorgBadHashes(t, true) } func testReorgBadHashes(t *testing.T, full bool) { - bc := newTestBlockChain(true) - defer bc.Stop() - + // Create a pristine chain and database + db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } // Create a chain, import and ban afterwards - headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10) - blocks := makeBlockChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10) + headers := makeHeaderChain(blockchain.CurrentHeader(), 4, ethash.NewFaker(), db, 10) + blocks := makeBlockChain(blockchain.CurrentBlock(), 4, ethash.NewFaker(), db, 10) if full { - if _, err := bc.InsertChain(blocks); err != nil { - t.Fatalf("failed to import blocks: %v", err) + if _, err = blockchain.InsertChain(blocks); err != nil { + t.Errorf("failed to import blocks: %v", err) } - if bc.CurrentBlock().Hash() != blocks[3].Hash() { - t.Errorf("last block hash mismatch: have: %x, want %x", bc.CurrentBlock().Hash(), blocks[3].Header().Hash()) + if blockchain.CurrentBlock().Hash() != blocks[3].Hash() { + t.Errorf("last block hash mismatch: have: %x, want %x", blockchain.CurrentBlock().Hash(), blocks[3].Header().Hash()) } BadHashes[blocks[3].Header().Hash()] = true defer func() { delete(BadHashes, blocks[3].Header().Hash()) }() } else { - if _, err := bc.InsertHeaderChain(headers, 1); err != nil { - t.Fatalf("failed to import headers: %v", err) + if _, err = blockchain.InsertHeaderChain(headers, 1); err != nil { + t.Errorf("failed to import headers: %v", err) } - if bc.CurrentHeader().Hash() != headers[3].Hash() { - t.Errorf("last header hash mismatch: have: %x, want %x", bc.CurrentHeader().Hash(), headers[3].Hash()) + if blockchain.CurrentHeader().Hash() != headers[3].Hash() { + t.Errorf("last header hash mismatch: have: %x, want %x", blockchain.CurrentHeader().Hash(), headers[3].Hash()) } BadHashes[headers[3].Hash()] = true defer func() { delete(BadHashes, headers[3].Hash()) }() } + blockchain.Stop() // Create a new BlockChain and check that it rolled back the state. - ncm, err := NewBlockChain(bc.db, nil, bc.chainConfig, ethash.NewFaker(), vm.Config{}) + ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{}) if err != nil { t.Fatalf("failed to create new chain manager: %v", err) } - defer ncm.Stop() - if full { if ncm.CurrentBlock().Hash() != blocks[2].Header().Hash() { t.Errorf("last block hash mismatch: have: %x, want %x", ncm.CurrentBlock().Hash(), blocks[2].Header().Hash()) @@ -514,6 +504,7 @@ func testReorgBadHashes(t *testing.T, full bool) { t.Errorf("last header hash mismatch: have: %x, want %x", ncm.CurrentHeader().Hash(), headers[2].Hash()) } } + ncm.Stop() } // Tests chain insertions in the face of one entity containing an invalid nonce. @@ -989,10 +980,13 @@ done: // Tests if the canonical block can be fetched from the database during chain insertion. func TestCanonicalBlockRetrieval(t *testing.T) { - bc := newTestBlockChain(true) - defer bc.Stop() + _, blockchain, err := newCanonical(ethash.NewFaker(), 0, true) + if err != nil { + t.Fatalf("failed to create pristine chain: %v", err) + } + defer blockchain.Stop() - chain, _ := GenerateChain(bc.chainConfig, bc.genesisBlock, ethash.NewFaker(), bc.db, 10, func(i int, gen *BlockGen) {}) + chain, _ := GenerateChain(blockchain.chainConfig, blockchain.genesisBlock, ethash.NewFaker(), blockchain.db, 10, func(i int, gen *BlockGen) {}) var pend sync.WaitGroup pend.Add(len(chain)) @@ -1003,14 +997,14 @@ func TestCanonicalBlockRetrieval(t *testing.T) { // try to retrieve a block by its canonical hash and see if the block data can be retrieved. for { - ch := GetCanonicalHash(bc.db, block.NumberU64()) + ch := GetCanonicalHash(blockchain.db, block.NumberU64()) if ch == (common.Hash{}) { continue // busy wait for canonical hash to be written } if ch != block.Hash() { t.Fatalf("unknown canonical hash, want %s, got %s", block.Hash().Hex(), ch.Hex()) } - fb := GetBlock(bc.db, ch, block.NumberU64()) + fb := GetBlock(blockchain.db, ch, block.NumberU64()) if fb == nil { t.Fatalf("unable to retrieve block %d for canonical hash: %s", block.NumberU64(), ch.Hex()) } @@ -1021,7 +1015,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) { } }(chain[i]) - if _, err := bc.InsertChain(types.Blocks{chain[i]}); err != nil { + if _, err := blockchain.InsertChain(types.Blocks{chain[i]}); err != nil { t.Fatalf("failed to insert block %d: %v", i, err) } } diff --git a/core/database_util.go b/core/database_util.go index c6b125dae9..61ab701349 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -70,8 +70,8 @@ var ( ErrChainConfigNotFound = errors.New("ChainConfig not found") // general config not found error - preimageCounter = metrics.NewCounter("db/preimage/total") - preimageHitCounter = metrics.NewCounter("db/preimage/hits") + preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) + preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) ) // TxLookupEntry is a positional metadata to help looking up the data content of diff --git a/core/genesis_test.go b/core/genesis_test.go index cd548d4b1d..052ded6991 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -118,10 +118,12 @@ func TestSetupGenesis(t *testing.T) { // Commit the 'old' genesis block with Homestead transition at #2. // Advance to block #4, past the homestead transition block of customg. genesis := oldcustomg.MustCommit(db) + bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{}) defer bc.Stop() - bc.SetValidator(bproc{}) - bc.InsertChain(makeBlockChainWithDiff(genesis, []int{2, 3, 4, 5}, 0)) + + blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil) + bc.InsertChain(blocks) bc.CurrentBlock() // This should return a compatibility error. return SetupGenesisBlock(db, &customg) diff --git a/core/headerchain.go b/core/headerchain.go index 0e5215293d..73cd5d2c41 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/hashicorp/golang-lru" + "sync/atomic" ) const ( @@ -51,8 +52,8 @@ type HeaderChain struct { chainDb ethdb.Database genesisHeader *types.Header - currentHeader *types.Header // Current head of the header chain (may be above the block chain!) - currentHeaderHash common.Hash // Hash of the current head of the header chain (prevent recomputing all the time) + currentHeader atomic.Value // Current head of the header chain (may be above the block chain!) + currentHeaderHash common.Hash // Hash of the current head of the header chain (prevent recomputing all the time) headerCache *lru.Cache // Cache for the most recent block headers tdCache *lru.Cache // Cache for the most recent block total difficulties @@ -95,13 +96,13 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c return nil, ErrNoGenesis } - hc.currentHeader = hc.genesisHeader + hc.currentHeader.Store(hc.genesisHeader) if head := GetHeadBlockHash(chainDb); head != (common.Hash{}) { if chead := hc.GetHeaderByHash(head); chead != nil { - hc.currentHeader = chead + hc.currentHeader.Store(chead) } } - hc.currentHeaderHash = hc.currentHeader.Hash() + hc.currentHeaderHash = hc.CurrentHeader().Hash() return hc, nil } @@ -139,7 +140,7 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er if ptd == nil { return NonStatTy, consensus.ErrUnknownAncestor } - localTd := hc.GetTd(hc.currentHeaderHash, hc.currentHeader.Number.Uint64()) + localTd := hc.GetTd(hc.currentHeaderHash, hc.CurrentHeader().Number.Uint64()) externTd := new(big.Int).Add(header.Difficulty, ptd) // Irrelevant of the canonical status, write the td and header to the database @@ -181,7 +182,8 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er if err := WriteHeadHeaderHash(hc.chainDb, hash); err != nil { log.Crit("Failed to insert head header hash", "err", err) } - hc.currentHeaderHash, hc.currentHeader = hash, types.CopyHeader(header) + hc.currentHeaderHash = hash + hc.currentHeader.Store(types.CopyHeader(header)) status = CanonStatTy } else { @@ -383,7 +385,7 @@ func (hc *HeaderChain) GetHeaderByNumber(number uint64) *types.Header { // CurrentHeader retrieves the current head header of the canonical chain. The // header is retrieved from the HeaderChain's internal cache. func (hc *HeaderChain) CurrentHeader() *types.Header { - return hc.currentHeader + return hc.currentHeader.Load().(*types.Header) } // SetCurrentHeader sets the current head header of the canonical chain. @@ -391,7 +393,7 @@ func (hc *HeaderChain) SetCurrentHeader(head *types.Header) { if err := WriteHeadHeaderHash(hc.chainDb, head.Hash()); err != nil { log.Crit("Failed to insert head header hash", "err", err) } - hc.currentHeader = head + hc.currentHeader.Store(head) hc.currentHeaderHash = head.Hash() } @@ -403,19 +405,20 @@ type DeleteCallback func(common.Hash, uint64) // will be deleted and the new one set. func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { height := uint64(0) - if hc.currentHeader != nil { - height = hc.currentHeader.Number.Uint64() + + if hdr := hc.CurrentHeader(); hdr != nil { + height = hdr.Number.Uint64() } - for hc.currentHeader != nil && hc.currentHeader.Number.Uint64() > head { - hash := hc.currentHeader.Hash() - num := hc.currentHeader.Number.Uint64() + for hdr := hc.CurrentHeader(); hdr != nil && hdr.Number.Uint64() > head; hdr = hc.CurrentHeader() { + hash := hdr.Hash() + num := hdr.Number.Uint64() if delFn != nil { delFn(hash, num) } DeleteHeader(hc.chainDb, hash, num) DeleteTd(hc.chainDb, hash, num) - hc.currentHeader = hc.GetHeader(hc.currentHeader.ParentHash, hc.currentHeader.Number.Uint64()-1) + hc.currentHeader.Store(hc.GetHeader(hdr.ParentHash, hdr.Number.Uint64()-1)) } // Roll back the canonical chain numbering for i := height; i > head; i-- { @@ -426,10 +429,10 @@ func (hc *HeaderChain) SetHead(head uint64, delFn DeleteCallback) { hc.tdCache.Purge() hc.numberCache.Purge() - if hc.currentHeader == nil { - hc.currentHeader = hc.genesisHeader + if hc.CurrentHeader() == nil { + hc.currentHeader.Store(hc.genesisHeader) } - hc.currentHeaderHash = hc.currentHeader.Hash() + hc.currentHeaderHash = hc.CurrentHeader().Hash() if err := WriteHeadHeaderHash(hc.chainDb, hc.currentHeaderHash); err != nil { log.Crit("Failed to reset head header hash", "err", err) diff --git a/core/tx_pool.go b/core/tx_pool.go index dc3ddc4232..0534fe57a1 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -87,20 +87,20 @@ var ( var ( // Metrics for the pending pool - pendingDiscardCounter = metrics.NewCounter("txpool/pending/discard") - pendingReplaceCounter = metrics.NewCounter("txpool/pending/replace") - pendingRateLimitCounter = metrics.NewCounter("txpool/pending/ratelimit") // Dropped due to rate limiting - pendingNofundsCounter = metrics.NewCounter("txpool/pending/nofunds") // Dropped due to out-of-funds + pendingDiscardCounter = metrics.NewRegisteredCounter("txpool/pending/discard", nil) + pendingReplaceCounter = metrics.NewRegisteredCounter("txpool/pending/replace", nil) + pendingRateLimitCounter = metrics.NewRegisteredCounter("txpool/pending/ratelimit", nil) // Dropped due to rate limiting + pendingNofundsCounter = metrics.NewRegisteredCounter("txpool/pending/nofunds", nil) // Dropped due to out-of-funds // Metrics for the queued pool - queuedDiscardCounter = metrics.NewCounter("txpool/queued/discard") - queuedReplaceCounter = metrics.NewCounter("txpool/queued/replace") - queuedRateLimitCounter = metrics.NewCounter("txpool/queued/ratelimit") // Dropped due to rate limiting - queuedNofundsCounter = metrics.NewCounter("txpool/queued/nofunds") // Dropped due to out-of-funds + queuedDiscardCounter = metrics.NewRegisteredCounter("txpool/queued/discard", nil) + queuedReplaceCounter = metrics.NewRegisteredCounter("txpool/queued/replace", nil) + queuedRateLimitCounter = metrics.NewRegisteredCounter("txpool/queued/ratelimit", nil) // Dropped due to rate limiting + queuedNofundsCounter = metrics.NewRegisteredCounter("txpool/queued/nofunds", nil) // Dropped due to out-of-funds // General tx metrics - invalidTxCounter = metrics.NewCounter("txpool/invalid") - underpricedTxCounter = metrics.NewCounter("txpool/underpriced") + invalidTxCounter = metrics.NewRegisteredCounter("txpool/invalid", nil) + underpricedTxCounter = metrics.NewRegisteredCounter("txpool/underpriced", nil) ) // TxStatus is the current status of a transaction as seen by the pool. diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 7661725010..6daf4e10df 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -302,6 +302,66 @@ func opMulmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *S return nil, nil } +// opSHL implements Shift Left +// The SHL instruction (shift left) pops 2 values from the stack, first arg1 and then arg2, +// and pushes on the stack arg2 shifted to the left by arg1 number of bits. +func opSHL(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + // Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards + shift, value := math.U256(stack.pop()), math.U256(stack.peek()) + defer evm.interpreter.intPool.put(shift) // First operand back into the pool + + if shift.Cmp(common.Big256) >= 0 { + value.SetUint64(0) + return nil, nil + } + n := uint(shift.Uint64()) + math.U256(value.Lsh(value, n)) + + return nil, nil +} + +// opSHR implements Logical Shift Right +// The SHR instruction (logical shift right) pops 2 values from the stack, first arg1 and then arg2, +// and pushes on the stack arg2 shifted to the right by arg1 number of bits with zero fill. +func opSHR(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + // Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards + shift, value := math.U256(stack.pop()), math.U256(stack.peek()) + defer evm.interpreter.intPool.put(shift) // First operand back into the pool + + if shift.Cmp(common.Big256) >= 0 { + value.SetUint64(0) + return nil, nil + } + n := uint(shift.Uint64()) + math.U256(value.Rsh(value, n)) + + return nil, nil +} + +// opSAR implements Arithmetic Shift Right +// The SAR instruction (arithmetic shift right) pops 2 values from the stack, first arg1 and then arg2, +// and pushes on the stack arg2 shifted to the right by arg1 number of bits with sign extension. +func opSAR(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { + // Note, S256 returns (potentially) a new bigint, so we're popping, not peeking this one + shift, value := math.U256(stack.pop()), math.S256(stack.pop()) + defer evm.interpreter.intPool.put(shift) // First operand back into the pool + + if shift.Cmp(common.Big256) >= 0 { + if value.Sign() > 0 { + value.SetUint64(0) + } else { + value.SetInt64(-1) + } + stack.push(math.U256(value)) + return nil, nil + } + n := uint(shift.Uint64()) + value.Rsh(value, n) + stack.push(math.U256(value)) + + return nil, nil +} + func opSha3(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) { offset, size := stack.pop(), stack.pop() data := memory.Get(offset.Int64(), size.Int64()) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 180433ea88..eef4328bde 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -24,6 +24,48 @@ import ( "github.com/ethereum/go-ethereum/params" ) +type twoOperandTest struct { + x string + y string + expected string +} + +func testTwoOperandOp(t *testing.T, tests []twoOperandTest, opFn func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error)) { + var ( + env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) + stack = newstack() + pc = uint64(0) + ) + for i, test := range tests { + x := new(big.Int).SetBytes(common.Hex2Bytes(test.x)) + shift := new(big.Int).SetBytes(common.Hex2Bytes(test.y)) + expected := new(big.Int).SetBytes(common.Hex2Bytes(test.expected)) + stack.push(x) + stack.push(shift) + opFn(&pc, env, nil, nil, stack) + actual := stack.pop() + if actual.Cmp(expected) != 0 { + t.Errorf("Testcase %d, expected %v, got %v", i, expected, actual) + } + // Check pool usage + // 1.pool is not allowed to contain anything on the stack + // 2.pool is not allowed to contain the same pointers twice + if env.interpreter.intPool.pool.len() > 0 { + + poolvals := make(map[*big.Int]struct{}) + poolvals[actual] = struct{}{} + + for env.interpreter.intPool.pool.len() > 0 { + key := env.interpreter.intPool.get() + if _, exist := poolvals[key]; exist { + t.Errorf("Testcase %d, pool contains double-entry", i) + } + poolvals[key] = struct{}{} + } + } + } +} + func TestByteOp(t *testing.T) { var ( env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) @@ -57,6 +99,98 @@ func TestByteOp(t *testing.T) { } } +func TestSHL(t *testing.T) { + // Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#shl-shift-left + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000002"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "ff", "8000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "0101", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "8000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"}, + } + testTwoOperandOp(t, tests, opSHL) +} + +func TestSHR(t *testing.T) { + // Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#shr-logical-shift-right + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "01", "4000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "ff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0101", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + } + testTwoOperandOp(t, tests, opSHR) +} + +func TestSAR(t *testing.T) { + // Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#sar-arithmetic-shift-right + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "01", "c000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "ff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0100", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"8000000000000000000000000000000000000000000000000000000000000000", "0101", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"}, + {"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"4000000000000000000000000000000000000000000000000000000000000000", "fe", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "f8", "000000000000000000000000000000000000000000000000000000000000007f"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "fe", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"}, + } + + testTwoOperandOp(t, tests, opSAR) +} + +func TestSGT(t *testing.T) { + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + } + testTwoOperandOp(t, tests, opSgt) +} + +func TestSLT(t *testing.T) { + tests := []twoOperandTest{ + {"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"0000000000000000000000000000000000000000000000000000000000000001", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"}, + {"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"}, + } + testTwoOperandOp(t, tests, opSlt) +} + func opBenchmark(bench *testing.B, op func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error), args ...string) { var ( env = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false}) @@ -259,3 +393,22 @@ func BenchmarkOpMulmod(b *testing.B) { opBenchmark(b, opMulmod, x, y, z) } + +func BenchmarkOpSHL(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + y := "ff" + + opBenchmark(b, opSHL, x, y) +} +func BenchmarkOpSHR(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + y := "ff" + + opBenchmark(b, opSHR, x, y) +} +func BenchmarkOpSAR(b *testing.B) { + x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff" + y := "ff" + + opBenchmark(b, opSAR, x, y) +} diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 82a6d3de6d..e12dd7e5a1 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -68,6 +68,8 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter { // we'll set the default jump table. if !cfg.JumpTable[STOP].valid { switch { + case evm.ChainConfig().IsConstantinople(evm.BlockNumber): + cfg.JumpTable = constantinopleInstructionSet case evm.ChainConfig().IsByzantium(evm.BlockNumber): cfg.JumpTable = byzantiumInstructionSet case evm.ChainConfig().IsHomestead(evm.BlockNumber): diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index a1c5ad9c67..3389941353 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -51,11 +51,38 @@ type operation struct { } var ( - frontierInstructionSet = NewFrontierInstructionSet() - homesteadInstructionSet = NewHomesteadInstructionSet() - byzantiumInstructionSet = NewByzantiumInstructionSet() + frontierInstructionSet = NewFrontierInstructionSet() + homesteadInstructionSet = NewHomesteadInstructionSet() + byzantiumInstructionSet = NewByzantiumInstructionSet() + constantinopleInstructionSet = NewConstantinopleInstructionSet() ) +// NewConstantinopleInstructionSet returns the frontier, homestead +// byzantium and contantinople instructions. +func NewConstantinopleInstructionSet() [256]operation { + // instructions that can be executed during the byzantium phase. + instructionSet := NewByzantiumInstructionSet() + instructionSet[SHL] = operation{ + execute: opSHL, + gasCost: constGasFunc(GasFastestStep), + validateStack: makeStackFunc(2, 1), + valid: true, + } + instructionSet[SHR] = operation{ + execute: opSHR, + gasCost: constGasFunc(GasFastestStep), + validateStack: makeStackFunc(2, 1), + valid: true, + } + instructionSet[SAR] = operation{ + execute: opSAR, + gasCost: constGasFunc(GasFastestStep), + validateStack: makeStackFunc(2, 1), + valid: true, + } + return instructionSet +} + // NewByzantiumInstructionSet returns the frontier, homestead and // byzantium instructions. func NewByzantiumInstructionSet() [256]operation { diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 0c65507355..7fe55b72f6 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -63,6 +63,9 @@ const ( XOR NOT BYTE + SHL + SHR + SAR SHA3 = 0x20 ) @@ -234,6 +237,9 @@ var opCodeToString = map[OpCode]string{ OR: "OR", XOR: "XOR", BYTE: "BYTE", + SHL: "SHL", + SHR: "SHR", + SAR: "SAR", ADDMOD: "ADDMOD", MULMOD: "MULMOD", @@ -400,6 +406,9 @@ var stringToOp = map[string]OpCode{ "OR": OR, "XOR": XOR, "BYTE": BYTE, + "SHL": SHL, + "SHR": SHR, + "SAR": SAR, "ADDMOD": ADDMOD, "MULMOD": MULMOD, "SHA3": SHA3, diff --git a/dashboard/dashboard.go b/dashboard/dashboard.go index fca38f6ae5..ce49701a5e 100644 --- a/dashboard/dashboard.go +++ b/dashboard/dashboard.go @@ -38,10 +38,10 @@ import ( "github.com/elastic/gosigar" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "github.com/rcrowley/go-metrics" "golang.org/x/net/websocket" ) diff --git a/eth/api_backend.go b/eth/api_backend.go index 91f392f94f..ecd5488a24 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -104,6 +104,18 @@ func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)), nil } +func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { + receipts := core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) + if receipts == nil { + return nil, nil + } + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs, nil +} + func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 7ede530a94..d13247766a 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -31,8 +31,8 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" - "github.com/rcrowley/go-metrics" ) var ( diff --git a/eth/downloader/metrics.go b/eth/downloader/metrics.go index 58764ccf06..d4eb337946 100644 --- a/eth/downloader/metrics.go +++ b/eth/downloader/metrics.go @@ -23,21 +23,21 @@ import ( ) var ( - headerInMeter = metrics.NewMeter("eth/downloader/headers/in") - headerReqTimer = metrics.NewTimer("eth/downloader/headers/req") - headerDropMeter = metrics.NewMeter("eth/downloader/headers/drop") - headerTimeoutMeter = metrics.NewMeter("eth/downloader/headers/timeout") + headerInMeter = metrics.NewRegisteredMeter("eth/downloader/headers/in", nil) + headerReqTimer = metrics.NewRegisteredTimer("eth/downloader/headers/req", nil) + headerDropMeter = metrics.NewRegisteredMeter("eth/downloader/headers/drop", nil) + headerTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/headers/timeout", nil) - bodyInMeter = metrics.NewMeter("eth/downloader/bodies/in") - bodyReqTimer = metrics.NewTimer("eth/downloader/bodies/req") - bodyDropMeter = metrics.NewMeter("eth/downloader/bodies/drop") - bodyTimeoutMeter = metrics.NewMeter("eth/downloader/bodies/timeout") + bodyInMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/in", nil) + bodyReqTimer = metrics.NewRegisteredTimer("eth/downloader/bodies/req", nil) + bodyDropMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/drop", nil) + bodyTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/timeout", nil) - receiptInMeter = metrics.NewMeter("eth/downloader/receipts/in") - receiptReqTimer = metrics.NewTimer("eth/downloader/receipts/req") - receiptDropMeter = metrics.NewMeter("eth/downloader/receipts/drop") - receiptTimeoutMeter = metrics.NewMeter("eth/downloader/receipts/timeout") + receiptInMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/in", nil) + receiptReqTimer = metrics.NewRegisteredTimer("eth/downloader/receipts/req", nil) + receiptDropMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/drop", nil) + receiptTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/timeout", nil) - stateInMeter = metrics.NewMeter("eth/downloader/states/in") - stateDropMeter = metrics.NewMeter("eth/downloader/states/drop") + stateInMeter = metrics.NewRegisteredMeter("eth/downloader/states/in", nil) + stateDropMeter = metrics.NewRegisteredMeter("eth/downloader/states/drop", nil) ) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index a1a70e46ea..359cce54b5 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -28,7 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" - "github.com/rcrowley/go-metrics" + "github.com/ethereum/go-ethereum/metrics" "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go index 1ed8075bf3..d68d12f000 100644 --- a/eth/fetcher/metrics.go +++ b/eth/fetcher/metrics.go @@ -23,21 +23,21 @@ import ( ) var ( - propAnnounceInMeter = metrics.NewMeter("eth/fetcher/prop/announces/in") - propAnnounceOutTimer = metrics.NewTimer("eth/fetcher/prop/announces/out") - propAnnounceDropMeter = metrics.NewMeter("eth/fetcher/prop/announces/drop") - propAnnounceDOSMeter = metrics.NewMeter("eth/fetcher/prop/announces/dos") + propAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/in", nil) + propAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/announces/out", nil) + propAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/drop", nil) + propAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/dos", nil) - propBroadcastInMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/in") - propBroadcastOutTimer = metrics.NewTimer("eth/fetcher/prop/broadcasts/out") - propBroadcastDropMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/drop") - propBroadcastDOSMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/dos") + propBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/in", nil) + propBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/broadcasts/out", nil) + propBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/drop", nil) + propBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/dos", nil) - headerFetchMeter = metrics.NewMeter("eth/fetcher/fetch/headers") - bodyFetchMeter = metrics.NewMeter("eth/fetcher/fetch/bodies") + headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/headers", nil) + bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/bodies", nil) - headerFilterInMeter = metrics.NewMeter("eth/fetcher/filter/headers/in") - headerFilterOutMeter = metrics.NewMeter("eth/fetcher/filter/headers/out") - bodyFilterInMeter = metrics.NewMeter("eth/fetcher/filter/bodies/in") - bodyFilterOutMeter = metrics.NewMeter("eth/fetcher/filter/bodies/out") + headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/in", nil) + headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/out", nil) + bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/in", nil) + bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/out", nil) ) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 43d7e2a812..5dfe60e772 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -34,6 +34,7 @@ type Backend interface { EventMux() *event.TypeMux HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription @@ -201,16 +202,28 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e // match the filter criteria. This function is called when the bloom filter signals a potential match. func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { // Get the logs of the block - receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + logsList, err := f.backend.GetLogs(ctx, header.Hash()) if err != nil { return nil, err } var unfiltered []*types.Log - for _, receipt := range receipts { - unfiltered = append(unfiltered, receipt.Logs...) + for _, logs := range logsList { + unfiltered = append(unfiltered, logs...) } logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) if len(logs) > 0 { + // We have matching logs, check if we need to resolve full logs via the light client + if logs[0].TxHash == (common.Hash{}) { + receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + if err != nil { + return nil, err + } + unfiltered = unfiltered[:0] + for _, receipt := range receipts { + unfiltered = append(unfiltered, receipt.Logs...) + } + logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) + } return logs, nil } return nil, nil diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index b09998f9c7..f8097c7b95 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -375,19 +375,35 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // Get the logs of the block ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - receipts, err := es.backend.GetReceipts(ctx, header.Hash()) + logsList, err := es.backend.GetLogs(ctx, header.Hash()) if err != nil { return nil } var unfiltered []*types.Log - for _, receipt := range receipts { - for _, log := range receipt.Logs { + for _, logs := range logsList { + for _, log := range logs { logcopy := *log logcopy.Removed = remove unfiltered = append(unfiltered, &logcopy) } } logs := filterLogs(unfiltered, nil, nil, addresses, topics) + if len(logs) > 0 && logs[0].TxHash == (common.Hash{}) { + // We have matching but non-derived logs + receipts, err := es.backend.GetReceipts(ctx, header.Hash()) + if err != nil { + return nil + } + unfiltered = unfiltered[:0] + for _, receipt := range receipts { + for _, log := range receipt.Logs { + logcopy := *log + logcopy.Removed = remove + unfiltered = append(unfiltered, &logcopy) + } + } + logs = filterLogs(unfiltered, nil, nil, addresses, topics) + } return logs } return nil diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 7ec3b4be7f..61761151a2 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -69,8 +69,19 @@ func (b *testBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumbe } func (b *testBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) { - num := core.GetBlockNumber(b.db, blockHash) - return core.GetBlockReceipts(b.db, blockHash, num), nil + number := core.GetBlockNumber(b.db, blockHash) + return core.GetBlockReceipts(b.db, blockHash, number), nil +} + +func (b *testBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { + number := core.GetBlockNumber(b.db, blockHash) + receipts := core.GetBlockReceipts(b.db, blockHash, number) + + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs, nil } func (b *testBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription { diff --git a/eth/metrics.go b/eth/metrics.go index 5fa2597d4d..0533a2a875 100644 --- a/eth/metrics.go +++ b/eth/metrics.go @@ -22,38 +22,38 @@ import ( ) var ( - propTxnInPacketsMeter = metrics.NewMeter("eth/prop/txns/in/packets") - propTxnInTrafficMeter = metrics.NewMeter("eth/prop/txns/in/traffic") - propTxnOutPacketsMeter = metrics.NewMeter("eth/prop/txns/out/packets") - propTxnOutTrafficMeter = metrics.NewMeter("eth/prop/txns/out/traffic") - propHashInPacketsMeter = metrics.NewMeter("eth/prop/hashes/in/packets") - propHashInTrafficMeter = metrics.NewMeter("eth/prop/hashes/in/traffic") - propHashOutPacketsMeter = metrics.NewMeter("eth/prop/hashes/out/packets") - propHashOutTrafficMeter = metrics.NewMeter("eth/prop/hashes/out/traffic") - propBlockInPacketsMeter = metrics.NewMeter("eth/prop/blocks/in/packets") - propBlockInTrafficMeter = metrics.NewMeter("eth/prop/blocks/in/traffic") - propBlockOutPacketsMeter = metrics.NewMeter("eth/prop/blocks/out/packets") - propBlockOutTrafficMeter = metrics.NewMeter("eth/prop/blocks/out/traffic") - reqHeaderInPacketsMeter = metrics.NewMeter("eth/req/headers/in/packets") - reqHeaderInTrafficMeter = metrics.NewMeter("eth/req/headers/in/traffic") - reqHeaderOutPacketsMeter = metrics.NewMeter("eth/req/headers/out/packets") - reqHeaderOutTrafficMeter = metrics.NewMeter("eth/req/headers/out/traffic") - reqBodyInPacketsMeter = metrics.NewMeter("eth/req/bodies/in/packets") - reqBodyInTrafficMeter = metrics.NewMeter("eth/req/bodies/in/traffic") - reqBodyOutPacketsMeter = metrics.NewMeter("eth/req/bodies/out/packets") - reqBodyOutTrafficMeter = metrics.NewMeter("eth/req/bodies/out/traffic") - reqStateInPacketsMeter = metrics.NewMeter("eth/req/states/in/packets") - reqStateInTrafficMeter = metrics.NewMeter("eth/req/states/in/traffic") - reqStateOutPacketsMeter = metrics.NewMeter("eth/req/states/out/packets") - reqStateOutTrafficMeter = metrics.NewMeter("eth/req/states/out/traffic") - reqReceiptInPacketsMeter = metrics.NewMeter("eth/req/receipts/in/packets") - reqReceiptInTrafficMeter = metrics.NewMeter("eth/req/receipts/in/traffic") - reqReceiptOutPacketsMeter = metrics.NewMeter("eth/req/receipts/out/packets") - reqReceiptOutTrafficMeter = metrics.NewMeter("eth/req/receipts/out/traffic") - miscInPacketsMeter = metrics.NewMeter("eth/misc/in/packets") - miscInTrafficMeter = metrics.NewMeter("eth/misc/in/traffic") - miscOutPacketsMeter = metrics.NewMeter("eth/misc/out/packets") - miscOutTrafficMeter = metrics.NewMeter("eth/misc/out/traffic") + propTxnInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/txns/in/packets", nil) + propTxnInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/txns/in/traffic", nil) + propTxnOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/txns/out/packets", nil) + propTxnOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/txns/out/traffic", nil) + propHashInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/hashes/in/packets", nil) + propHashInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/hashes/in/traffic", nil) + propHashOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/hashes/out/packets", nil) + propHashOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/hashes/out/traffic", nil) + propBlockInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/blocks/in/packets", nil) + propBlockInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/blocks/in/traffic", nil) + propBlockOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/blocks/out/packets", nil) + propBlockOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/blocks/out/traffic", nil) + reqHeaderInPacketsMeter = metrics.NewRegisteredMeter("eth/req/headers/in/packets", nil) + reqHeaderInTrafficMeter = metrics.NewRegisteredMeter("eth/req/headers/in/traffic", nil) + reqHeaderOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/headers/out/packets", nil) + reqHeaderOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/headers/out/traffic", nil) + reqBodyInPacketsMeter = metrics.NewRegisteredMeter("eth/req/bodies/in/packets", nil) + reqBodyInTrafficMeter = metrics.NewRegisteredMeter("eth/req/bodies/in/traffic", nil) + reqBodyOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/bodies/out/packets", nil) + reqBodyOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/bodies/out/traffic", nil) + reqStateInPacketsMeter = metrics.NewRegisteredMeter("eth/req/states/in/packets", nil) + reqStateInTrafficMeter = metrics.NewRegisteredMeter("eth/req/states/in/traffic", nil) + reqStateOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/states/out/packets", nil) + reqStateOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/states/out/traffic", nil) + reqReceiptInPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/in/packets", nil) + reqReceiptInTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/in/traffic", nil) + reqReceiptOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/packets", nil) + reqReceiptOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/traffic", nil) + miscInPacketsMeter = metrics.NewRegisteredMeter("eth/misc/in/packets", nil) + miscInTrafficMeter = metrics.NewRegisteredMeter("eth/misc/in/traffic", nil) + miscOutPacketsMeter = metrics.NewRegisteredMeter("eth/misc/out/packets", nil) + miscOutTrafficMeter = metrics.NewRegisteredMeter("eth/misc/out/traffic", nil) ) // meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of diff --git a/ethdb/database.go b/ethdb/database.go index d86585f079..57d38f7f5f 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -29,8 +29,6 @@ import ( "github.com/syndtr/goleveldb/leveldb/filter" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" - - gometrics "github.com/rcrowley/go-metrics" ) var OpenFileLimit = 64 @@ -39,15 +37,15 @@ type LDBDatabase struct { fn string // filename for reporting db *leveldb.DB // LevelDB instance - getTimer gometrics.Timer // Timer for measuring the database get request counts and latencies - putTimer gometrics.Timer // Timer for measuring the database put request counts and latencies - delTimer gometrics.Timer // Timer for measuring the database delete request counts and latencies - missMeter gometrics.Meter // Meter for measuring the missed database get requests - readMeter gometrics.Meter // Meter for measuring the database get request data usage - writeMeter gometrics.Meter // Meter for measuring the database put request data usage - compTimeMeter gometrics.Meter // Meter for measuring the total time spent in database compaction - compReadMeter gometrics.Meter // Meter for measuring the data read during compaction - compWriteMeter gometrics.Meter // Meter for measuring the data written during compaction + getTimer metrics.Timer // Timer for measuring the database get request counts and latencies + putTimer metrics.Timer // Timer for measuring the database put request counts and latencies + delTimer metrics.Timer // Timer for measuring the database delete request counts and latencies + missMeter metrics.Meter // Meter for measuring the missed database get requests + readMeter metrics.Meter // Meter for measuring the database get request data usage + writeMeter metrics.Meter // Meter for measuring the database put request data usage + compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction + compReadMeter metrics.Meter // Meter for measuring the data read during compaction + compWriteMeter metrics.Meter // Meter for measuring the data written during compaction quitLock sync.Mutex // Mutex protecting the quit channel access quitChan chan chan error // Quit channel to stop the metrics collection before closing the database @@ -180,15 +178,15 @@ func (db *LDBDatabase) Meter(prefix string) { return } // Initialize all the metrics collector at the requested prefix - db.getTimer = metrics.NewTimer(prefix + "user/gets") - db.putTimer = metrics.NewTimer(prefix + "user/puts") - db.delTimer = metrics.NewTimer(prefix + "user/dels") - db.missMeter = metrics.NewMeter(prefix + "user/misses") - db.readMeter = metrics.NewMeter(prefix + "user/reads") - db.writeMeter = metrics.NewMeter(prefix + "user/writes") - db.compTimeMeter = metrics.NewMeter(prefix + "compact/time") - db.compReadMeter = metrics.NewMeter(prefix + "compact/input") - db.compWriteMeter = metrics.NewMeter(prefix + "compact/output") + db.getTimer = metrics.NewRegisteredTimer(prefix+"user/gets", nil) + db.putTimer = metrics.NewRegisteredTimer(prefix+"user/puts", nil) + db.delTimer = metrics.NewRegisteredTimer(prefix+"user/dels", nil) + db.missMeter = metrics.NewRegisteredMeter(prefix+"user/misses", nil) + db.readMeter = metrics.NewRegisteredMeter(prefix+"user/reads", nil) + db.writeMeter = metrics.NewRegisteredMeter(prefix+"user/writes", nil) + db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil) + db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil) + db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil) // Create a quit channel for the periodic collector and run it db.quitLock.Lock() diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 6247cc7dc0..1f181bf8b0 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -26,6 +26,8 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log/term" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/metrics/exp" colorable "github.com/mattn/go-colorable" "gopkg.in/urfave/cli.v1" ) @@ -127,6 +129,10 @@ func Setup(ctx *cli.Context) error { // pprof server if ctx.GlobalBool(pprofFlag.Name) { + // Hook go-metrics into expvar on any /debug/metrics request, load all vars + // from the registry into expvar, and execute regular expvar handler. + exp.Exp(metrics.DefaultRegistry) + address := fmt.Sprintf("%s:%d", ctx.GlobalString(pprofAddrFlag.Name), ctx.GlobalInt(pprofPortFlag.Name)) go func() { log.Info("Starting pprof server", "addr", fmt.Sprintf("http://%s/debug/pprof", address)) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 636d0bfe26..d021b127c5 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1032,15 +1032,19 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context, } // GetTransactionReceipt returns the transaction receipt for the given transaction hash. -func (s *PublicTransactionPoolAPI) GetTransactionReceipt(hash common.Hash) (map[string]interface{}, error) { +func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash) if tx == nil { return nil, errors.New("unknown transaction") } - receipt, _, _, _ := core.GetReceipt(s.b.ChainDb(), hash) // Old receipts don't have the lookup data available - if receipt == nil { + receipts, err := s.b.GetReceipts(ctx, blockHash) + if err != nil { + return nil, err + } + if len(receipts) <= int(index) { return nil, errors.New("unknown receipt") } + receipt := receipts[index] var signer types.Signer = types.FrontierSigner{} if tx.Protected() { diff --git a/les/api_backend.go b/les/api_backend.go index 56f617a7db..3fc5c33a44 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -87,6 +87,10 @@ func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash) return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) } +func (b *LesApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { + return light.GetBlockLogs(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)) +} + func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } diff --git a/les/metrics.go b/les/metrics.go index 0162a1d1ad..c282a62a1a 100644 --- a/les/metrics.go +++ b/les/metrics.go @@ -58,10 +58,10 @@ var ( reqReceiptInTrafficMeter = metrics.NewMeter("eth/req/receipts/in/traffic") reqReceiptOutPacketsMeter = metrics.NewMeter("eth/req/receipts/out/packets") reqReceiptOutTrafficMeter = metrics.NewMeter("eth/req/receipts/out/traffic")*/ - miscInPacketsMeter = metrics.NewMeter("les/misc/in/packets") - miscInTrafficMeter = metrics.NewMeter("les/misc/in/traffic") - miscOutPacketsMeter = metrics.NewMeter("les/misc/out/packets") - miscOutTrafficMeter = metrics.NewMeter("les/misc/out/traffic") + miscInPacketsMeter = metrics.NewRegisteredMeter("les/misc/in/packets", nil) + miscInTrafficMeter = metrics.NewRegisteredMeter("les/misc/in/traffic", nil) + miscOutPacketsMeter = metrics.NewRegisteredMeter("les/misc/out/packets", nil) + miscOutTrafficMeter = metrics.NewRegisteredMeter("les/misc/out/traffic", nil) ) // meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of diff --git a/light/lightchain.go b/light/lightchain.go index 181a1c2a62..2784615d35 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -171,9 +171,6 @@ func (bc *LightChain) SetHead(head uint64) { // GasLimit returns the gas limit of the current HEAD block. func (self *LightChain) GasLimit() uint64 { - self.mu.RLock() - defer self.mu.RUnlock() - return self.hc.CurrentHeader().GasLimit } @@ -387,9 +384,6 @@ func (self *LightChain) InsertHeaderChain(chain []*types.Header, checkFreq int) // CurrentHeader retrieves the current head header of the canonical chain. The // header is retrieved from the HeaderChain's internal cache. func (self *LightChain) CurrentHeader() *types.Header { - self.mu.RLock() - defer self.mu.RUnlock() - return self.hc.CurrentHeader() } diff --git a/light/odr_util.go b/light/odr_util.go index 8f92d64426..97ba440ace 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -126,15 +126,48 @@ func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint // GetBlockReceipts retrieves the receipts generated by the transactions included // in a block given by its hash. func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (types.Receipts, error) { + // Retrieve the potentially incomplete receipts from disk or network receipts := core.GetBlockReceipts(odr.Database(), hash, number) - if receipts != nil { - return receipts, nil + if receipts == nil { + r := &ReceiptsRequest{Hash: hash, Number: number} + if err := odr.Retrieve(ctx, r); err != nil { + return nil, err + } + receipts = r.Receipts } - r := &ReceiptsRequest{Hash: hash, Number: number} - if err := odr.Retrieve(ctx, r); err != nil { - return nil, err + // If the receipts are incomplete, fill the derived fields + if len(receipts) > 0 && receipts[0].TxHash == (common.Hash{}) { + block, err := GetBlock(ctx, odr, hash, number) + if err != nil { + return nil, err + } + genesis := core.GetCanonicalHash(odr.Database(), 0) + config, _ := core.GetChainConfig(odr.Database(), genesis) + + core.SetReceiptsData(config, block, receipts) + core.WriteBlockReceipts(odr.Database(), hash, number, receipts) } - return r.Receipts, nil + return receipts, nil +} + +// GetBlockLogs retrieves the logs generated by the transactions included in a +// block given by its hash. +func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) ([][]*types.Log, error) { + // Retrieve the potentially incomplete receipts from disk or network + receipts := core.GetBlockReceipts(odr.Database(), hash, number) + if receipts == nil { + r := &ReceiptsRequest{Hash: hash, Number: number} + if err := odr.Retrieve(ctx, r); err != nil { + return nil, err + } + receipts = r.Receipts + } + // Return the logs without deriving any computed fields on the receipts + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs, nil } // GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to the given bit index and section indexes diff --git a/metrics/FORK.md b/metrics/FORK.md new file mode 100644 index 0000000000..b19985bf56 --- /dev/null +++ b/metrics/FORK.md @@ -0,0 +1 @@ +This repo has been forked from https://github.com/rcrowley/go-metrics at commit e181e09 diff --git a/vendor/github.com/rcrowley/go-metrics/LICENSE b/metrics/LICENSE similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/LICENSE rename to metrics/LICENSE diff --git a/vendor/github.com/rcrowley/go-metrics/README.md b/metrics/README.md similarity index 81% rename from vendor/github.com/rcrowley/go-metrics/README.md rename to metrics/README.md index 2d1a6dcfa4..bc2a45a838 100644 --- a/vendor/github.com/rcrowley/go-metrics/README.md +++ b/metrics/README.md @@ -42,12 +42,22 @@ t.Update(47) Register() is not threadsafe. For threadsafe metric registration use GetOrRegister: -``` +```go t := metrics.GetOrRegisterTimer("account.create.latency", nil) t.Time(func() {}) t.Update(47) ``` +**NOTE:** Be sure to unregister short-lived meters and timers otherwise they will +leak memory: + +```go +// Will call Stop() on the Meter to allow for garbage collection +metrics.Unregister("quux") +// Or similarly for a Timer that embeds a Meter +metrics.Unregister("bang") +``` + Periodically log every metric in human-readable form to standard error: ```go @@ -81,12 +91,13 @@ issues [#121](https://github.com/rcrowley/go-metrics/issues/121) and ```go import "github.com/vrischmann/go-metrics-influxdb" -go influxdb.Influxdb(metrics.DefaultRegistry, 10e9, &influxdb.Config{ - Host: "127.0.0.1:8086", - Database: "metrics", - Username: "test", - Password: "test", -}) +go influxdb.InfluxDB(metrics.DefaultRegistry, + 10e9, + "127.0.0.1:8086", + "database-name", + "username", + "password" +) ``` Periodically upload every metric to Librato using the [Librato client](https://github.com/mihasya/go-metrics-librato): @@ -146,8 +157,10 @@ Publishing Metrics Clients are available for the following destinations: -* Librato - [https://github.com/mihasya/go-metrics-librato](https://github.com/mihasya/go-metrics-librato) -* Graphite - [https://github.com/cyberdelia/go-metrics-graphite](https://github.com/cyberdelia/go-metrics-graphite) -* InfluxDB - [https://github.com/vrischmann/go-metrics-influxdb](https://github.com/vrischmann/go-metrics-influxdb) -* Ganglia - [https://github.com/appscode/metlia](https://github.com/appscode/metlia) -* Prometheus - [https://github.com/deathowl/go-metrics-prometheus](https://github.com/deathowl/go-metrics-prometheus) +* Librato - https://github.com/mihasya/go-metrics-librato +* Graphite - https://github.com/cyberdelia/go-metrics-graphite +* InfluxDB - https://github.com/vrischmann/go-metrics-influxdb +* Ganglia - https://github.com/appscode/metlia +* Prometheus - https://github.com/deathowl/go-metrics-prometheus +* DataDog - https://github.com/syntaqx/go-metrics-datadog +* SignalFX - https://github.com/pascallouisperez/go-metrics-signalfx diff --git a/vendor/github.com/rcrowley/go-metrics/counter.go b/metrics/counter.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/counter.go rename to metrics/counter.go index bb7b039cb5..c7f2b4bd3a 100644 --- a/vendor/github.com/rcrowley/go-metrics/counter.go +++ b/metrics/counter.go @@ -22,7 +22,7 @@ func GetOrRegisterCounter(name string, r Registry) Counter { // NewCounter constructs a new StandardCounter. func NewCounter() Counter { - if UseNilMetrics { + if !Enabled { return NilCounter{} } return &StandardCounter{0} diff --git a/metrics/counter_test.go b/metrics/counter_test.go new file mode 100644 index 0000000000..dfb03b4e88 --- /dev/null +++ b/metrics/counter_test.go @@ -0,0 +1,77 @@ +package metrics + +import "testing" + +func BenchmarkCounter(b *testing.B) { + c := NewCounter() + b.ResetTimer() + for i := 0; i < b.N; i++ { + c.Inc(1) + } +} + +func TestCounterClear(t *testing.T) { + c := NewCounter() + c.Inc(1) + c.Clear() + if count := c.Count(); 0 != count { + t.Errorf("c.Count(): 0 != %v\n", count) + } +} + +func TestCounterDec1(t *testing.T) { + c := NewCounter() + c.Dec(1) + if count := c.Count(); -1 != count { + t.Errorf("c.Count(): -1 != %v\n", count) + } +} + +func TestCounterDec2(t *testing.T) { + c := NewCounter() + c.Dec(2) + if count := c.Count(); -2 != count { + t.Errorf("c.Count(): -2 != %v\n", count) + } +} + +func TestCounterInc1(t *testing.T) { + c := NewCounter() + c.Inc(1) + if count := c.Count(); 1 != count { + t.Errorf("c.Count(): 1 != %v\n", count) + } +} + +func TestCounterInc2(t *testing.T) { + c := NewCounter() + c.Inc(2) + if count := c.Count(); 2 != count { + t.Errorf("c.Count(): 2 != %v\n", count) + } +} + +func TestCounterSnapshot(t *testing.T) { + c := NewCounter() + c.Inc(1) + snapshot := c.Snapshot() + c.Inc(1) + if count := snapshot.Count(); 1 != count { + t.Errorf("c.Count(): 1 != %v\n", count) + } +} + +func TestCounterZero(t *testing.T) { + c := NewCounter() + if count := c.Count(); 0 != count { + t.Errorf("c.Count(): 0 != %v\n", count) + } +} + +func TestGetOrRegisterCounter(t *testing.T) { + r := NewRegistry() + NewRegisteredCounter("foo", r).Inc(47) + if c := GetOrRegisterCounter("foo", r); 47 != c.Count() { + t.Fatal(c) + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/debug.go b/metrics/debug.go similarity index 94% rename from vendor/github.com/rcrowley/go-metrics/debug.go rename to metrics/debug.go index 043ccefab6..de4a2739fe 100644 --- a/vendor/github.com/rcrowley/go-metrics/debug.go +++ b/metrics/debug.go @@ -22,7 +22,7 @@ var ( // Capture new values for the Go garbage collector statistics exported in // debug.GCStats. This is designed to be called as a goroutine. func CaptureDebugGCStats(r Registry, d time.Duration) { - for _ = range time.Tick(d) { + for range time.Tick(d) { CaptureDebugGCStatsOnce(r) } } @@ -41,8 +41,8 @@ func CaptureDebugGCStatsOnce(r Registry) { debug.ReadGCStats(&gcStats) debugMetrics.ReadGCStats.UpdateSince(t) - debugMetrics.GCStats.LastGC.Update(int64(gcStats.LastGC.UnixNano())) - debugMetrics.GCStats.NumGC.Update(int64(gcStats.NumGC)) + debugMetrics.GCStats.LastGC.Update(gcStats.LastGC.UnixNano()) + debugMetrics.GCStats.NumGC.Update(gcStats.NumGC) if lastGC != gcStats.LastGC && 0 < len(gcStats.Pause) { debugMetrics.GCStats.Pause.Update(int64(gcStats.Pause[0])) } diff --git a/metrics/debug_test.go b/metrics/debug_test.go new file mode 100644 index 0000000000..07eb867841 --- /dev/null +++ b/metrics/debug_test.go @@ -0,0 +1,48 @@ +package metrics + +import ( + "runtime" + "runtime/debug" + "testing" + "time" +) + +func BenchmarkDebugGCStats(b *testing.B) { + r := NewRegistry() + RegisterDebugGCStats(r) + b.ResetTimer() + for i := 0; i < b.N; i++ { + CaptureDebugGCStatsOnce(r) + } +} + +func TestDebugGCStatsBlocking(t *testing.T) { + if g := runtime.GOMAXPROCS(0); g < 2 { + t.Skipf("skipping TestDebugGCMemStatsBlocking with GOMAXPROCS=%d\n", g) + return + } + ch := make(chan int) + go testDebugGCStatsBlocking(ch) + var gcStats debug.GCStats + t0 := time.Now() + debug.ReadGCStats(&gcStats) + t1 := time.Now() + t.Log("i++ during debug.ReadGCStats:", <-ch) + go testDebugGCStatsBlocking(ch) + d := t1.Sub(t0) + t.Log(d) + time.Sleep(d) + t.Log("i++ during time.Sleep:", <-ch) +} + +func testDebugGCStatsBlocking(ch chan int) { + i := 0 + for { + select { + case ch <- i: + return + default: + i++ + } + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/ewma.go b/metrics/ewma.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/ewma.go rename to metrics/ewma.go index 694a1d0330..3aecd4fa35 100644 --- a/vendor/github.com/rcrowley/go-metrics/ewma.go +++ b/metrics/ewma.go @@ -17,7 +17,7 @@ type EWMA interface { // NewEWMA constructs a new EWMA with the given alpha. func NewEWMA(alpha float64) EWMA { - if UseNilMetrics { + if !Enabled { return NilEWMA{} } return &StandardEWMA{alpha: alpha} diff --git a/metrics/ewma_test.go b/metrics/ewma_test.go new file mode 100644 index 0000000000..0430fbd247 --- /dev/null +++ b/metrics/ewma_test.go @@ -0,0 +1,225 @@ +package metrics + +import "testing" + +func BenchmarkEWMA(b *testing.B) { + a := NewEWMA1() + b.ResetTimer() + for i := 0; i < b.N; i++ { + a.Update(1) + a.Tick() + } +} + +func TestEWMA1(t *testing.T) { + a := NewEWMA1() + a.Update(3) + a.Tick() + if rate := a.Rate(); 0.6 != rate { + t.Errorf("initial a.Rate(): 0.6 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.22072766470286553 != rate { + t.Errorf("1 minute a.Rate(): 0.22072766470286553 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.08120116994196772 != rate { + t.Errorf("2 minute a.Rate(): 0.08120116994196772 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.029872241020718428 != rate { + t.Errorf("3 minute a.Rate(): 0.029872241020718428 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.01098938333324054 != rate { + t.Errorf("4 minute a.Rate(): 0.01098938333324054 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.004042768199451294 != rate { + t.Errorf("5 minute a.Rate(): 0.004042768199451294 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.0014872513059998212 != rate { + t.Errorf("6 minute a.Rate(): 0.0014872513059998212 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.0005471291793327122 != rate { + t.Errorf("7 minute a.Rate(): 0.0005471291793327122 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.00020127757674150815 != rate { + t.Errorf("8 minute a.Rate(): 0.00020127757674150815 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 7.404588245200814e-05 != rate { + t.Errorf("9 minute a.Rate(): 7.404588245200814e-05 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 2.7239957857491083e-05 != rate { + t.Errorf("10 minute a.Rate(): 2.7239957857491083e-05 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 1.0021020474147462e-05 != rate { + t.Errorf("11 minute a.Rate(): 1.0021020474147462e-05 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 3.6865274119969525e-06 != rate { + t.Errorf("12 minute a.Rate(): 3.6865274119969525e-06 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 1.3561976441886433e-06 != rate { + t.Errorf("13 minute a.Rate(): 1.3561976441886433e-06 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 4.989172314621449e-07 != rate { + t.Errorf("14 minute a.Rate(): 4.989172314621449e-07 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 1.8354139230109722e-07 != rate { + t.Errorf("15 minute a.Rate(): 1.8354139230109722e-07 != %v\n", rate) + } +} + +func TestEWMA5(t *testing.T) { + a := NewEWMA5() + a.Update(3) + a.Tick() + if rate := a.Rate(); 0.6 != rate { + t.Errorf("initial a.Rate(): 0.6 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.49123845184678905 != rate { + t.Errorf("1 minute a.Rate(): 0.49123845184678905 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.4021920276213837 != rate { + t.Errorf("2 minute a.Rate(): 0.4021920276213837 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.32928698165641596 != rate { + t.Errorf("3 minute a.Rate(): 0.32928698165641596 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.269597378470333 != rate { + t.Errorf("4 minute a.Rate(): 0.269597378470333 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.2207276647028654 != rate { + t.Errorf("5 minute a.Rate(): 0.2207276647028654 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.18071652714732128 != rate { + t.Errorf("6 minute a.Rate(): 0.18071652714732128 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.14795817836496392 != rate { + t.Errorf("7 minute a.Rate(): 0.14795817836496392 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.12113791079679326 != rate { + t.Errorf("8 minute a.Rate(): 0.12113791079679326 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.09917933293295193 != rate { + t.Errorf("9 minute a.Rate(): 0.09917933293295193 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.08120116994196763 != rate { + t.Errorf("10 minute a.Rate(): 0.08120116994196763 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.06648189501740036 != rate { + t.Errorf("11 minute a.Rate(): 0.06648189501740036 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.05443077197364752 != rate { + t.Errorf("12 minute a.Rate(): 0.05443077197364752 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.04456414692860035 != rate { + t.Errorf("13 minute a.Rate(): 0.04456414692860035 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.03648603757513079 != rate { + t.Errorf("14 minute a.Rate(): 0.03648603757513079 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.0298722410207183831020718428 != rate { + t.Errorf("15 minute a.Rate(): 0.0298722410207183831020718428 != %v\n", rate) + } +} + +func TestEWMA15(t *testing.T) { + a := NewEWMA15() + a.Update(3) + a.Tick() + if rate := a.Rate(); 0.6 != rate { + t.Errorf("initial a.Rate(): 0.6 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.5613041910189706 != rate { + t.Errorf("1 minute a.Rate(): 0.5613041910189706 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.5251039914257684 != rate { + t.Errorf("2 minute a.Rate(): 0.5251039914257684 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.4912384518467888184678905 != rate { + t.Errorf("3 minute a.Rate(): 0.4912384518467888184678905 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.459557003018789 != rate { + t.Errorf("4 minute a.Rate(): 0.459557003018789 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.4299187863442732 != rate { + t.Errorf("5 minute a.Rate(): 0.4299187863442732 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.4021920276213831 != rate { + t.Errorf("6 minute a.Rate(): 0.4021920276213831 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.37625345116383313 != rate { + t.Errorf("7 minute a.Rate(): 0.37625345116383313 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.3519877317060185 != rate { + t.Errorf("8 minute a.Rate(): 0.3519877317060185 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.3292869816564153165641596 != rate { + t.Errorf("9 minute a.Rate(): 0.3292869816564153165641596 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.3080502714195546 != rate { + t.Errorf("10 minute a.Rate(): 0.3080502714195546 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.2881831806538789 != rate { + t.Errorf("11 minute a.Rate(): 0.2881831806538789 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.26959737847033216 != rate { + t.Errorf("12 minute a.Rate(): 0.26959737847033216 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.2522102307052083 != rate { + t.Errorf("13 minute a.Rate(): 0.2522102307052083 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.23594443252115815 != rate { + t.Errorf("14 minute a.Rate(): 0.23594443252115815 != %v\n", rate) + } + elapseMinute(a) + if rate := a.Rate(); 0.2207276647028646247028654470286553 != rate { + t.Errorf("15 minute a.Rate(): 0.2207276647028646247028654470286553 != %v\n", rate) + } +} + +func elapseMinute(a EWMA) { + for i := 0; i < 12; i++ { + a.Tick() + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/exp/exp.go b/metrics/exp/exp.go similarity index 73% rename from vendor/github.com/rcrowley/go-metrics/exp/exp.go rename to metrics/exp/exp.go index 11dd3f898a..c19d00a94d 100644 --- a/vendor/github.com/rcrowley/go-metrics/exp/exp.go +++ b/metrics/exp/exp.go @@ -8,7 +8,7 @@ import ( "net/http" "sync" - "github.com/rcrowley/go-metrics" + "github.com/ethereum/go-ethereum/metrics" ) type exp struct { @@ -97,22 +97,22 @@ func (exp *exp) publishHistogram(name string, metric metrics.Histogram) { exp.getInt(name + ".count").Set(h.Count()) exp.getFloat(name + ".min").Set(float64(h.Min())) exp.getFloat(name + ".max").Set(float64(h.Max())) - exp.getFloat(name + ".mean").Set(float64(h.Mean())) - exp.getFloat(name + ".std-dev").Set(float64(h.StdDev())) - exp.getFloat(name + ".50-percentile").Set(float64(ps[0])) - exp.getFloat(name + ".75-percentile").Set(float64(ps[1])) - exp.getFloat(name + ".95-percentile").Set(float64(ps[2])) - exp.getFloat(name + ".99-percentile").Set(float64(ps[3])) - exp.getFloat(name + ".999-percentile").Set(float64(ps[4])) + exp.getFloat(name + ".mean").Set(h.Mean()) + exp.getFloat(name + ".std-dev").Set(h.StdDev()) + exp.getFloat(name + ".50-percentile").Set(ps[0]) + exp.getFloat(name + ".75-percentile").Set(ps[1]) + exp.getFloat(name + ".95-percentile").Set(ps[2]) + exp.getFloat(name + ".99-percentile").Set(ps[3]) + exp.getFloat(name + ".999-percentile").Set(ps[4]) } func (exp *exp) publishMeter(name string, metric metrics.Meter) { m := metric.Snapshot() exp.getInt(name + ".count").Set(m.Count()) - exp.getFloat(name + ".one-minute").Set(float64(m.Rate1())) - exp.getFloat(name + ".five-minute").Set(float64(m.Rate5())) - exp.getFloat(name + ".fifteen-minute").Set(float64((m.Rate15()))) - exp.getFloat(name + ".mean").Set(float64(m.RateMean())) + exp.getFloat(name + ".one-minute").Set(m.Rate1()) + exp.getFloat(name + ".five-minute").Set(m.Rate5()) + exp.getFloat(name + ".fifteen-minute").Set((m.Rate15())) + exp.getFloat(name + ".mean").Set(m.RateMean()) } func (exp *exp) publishTimer(name string, metric metrics.Timer) { @@ -121,17 +121,17 @@ func (exp *exp) publishTimer(name string, metric metrics.Timer) { exp.getInt(name + ".count").Set(t.Count()) exp.getFloat(name + ".min").Set(float64(t.Min())) exp.getFloat(name + ".max").Set(float64(t.Max())) - exp.getFloat(name + ".mean").Set(float64(t.Mean())) - exp.getFloat(name + ".std-dev").Set(float64(t.StdDev())) - exp.getFloat(name + ".50-percentile").Set(float64(ps[0])) - exp.getFloat(name + ".75-percentile").Set(float64(ps[1])) - exp.getFloat(name + ".95-percentile").Set(float64(ps[2])) - exp.getFloat(name + ".99-percentile").Set(float64(ps[3])) - exp.getFloat(name + ".999-percentile").Set(float64(ps[4])) - exp.getFloat(name + ".one-minute").Set(float64(t.Rate1())) - exp.getFloat(name + ".five-minute").Set(float64(t.Rate5())) - exp.getFloat(name + ".fifteen-minute").Set(float64((t.Rate15()))) - exp.getFloat(name + ".mean-rate").Set(float64(t.RateMean())) + exp.getFloat(name + ".mean").Set(t.Mean()) + exp.getFloat(name + ".std-dev").Set(t.StdDev()) + exp.getFloat(name + ".50-percentile").Set(ps[0]) + exp.getFloat(name + ".75-percentile").Set(ps[1]) + exp.getFloat(name + ".95-percentile").Set(ps[2]) + exp.getFloat(name + ".99-percentile").Set(ps[3]) + exp.getFloat(name + ".999-percentile").Set(ps[4]) + exp.getFloat(name + ".one-minute").Set(t.Rate1()) + exp.getFloat(name + ".five-minute").Set(t.Rate5()) + exp.getFloat(name + ".fifteen-minute").Set(t.Rate15()) + exp.getFloat(name + ".mean-rate").Set(t.RateMean()) } func (exp *exp) syncToExpvar() { diff --git a/vendor/github.com/rcrowley/go-metrics/gauge.go b/metrics/gauge.go similarity index 98% rename from vendor/github.com/rcrowley/go-metrics/gauge.go rename to metrics/gauge.go index cb57a93889..0fbfdb8603 100644 --- a/vendor/github.com/rcrowley/go-metrics/gauge.go +++ b/metrics/gauge.go @@ -20,7 +20,7 @@ func GetOrRegisterGauge(name string, r Registry) Gauge { // NewGauge constructs a new StandardGauge. func NewGauge() Gauge { - if UseNilMetrics { + if !Enabled { return NilGauge{} } return &StandardGauge{0} @@ -38,7 +38,7 @@ func NewRegisteredGauge(name string, r Registry) Gauge { // NewFunctionalGauge constructs a new FunctionalGauge. func NewFunctionalGauge(f func() int64) Gauge { - if UseNilMetrics { + if !Enabled { return NilGauge{} } return &FunctionalGauge{value: f} diff --git a/vendor/github.com/rcrowley/go-metrics/gauge_float64.go b/metrics/gauge_float64.go similarity index 98% rename from vendor/github.com/rcrowley/go-metrics/gauge_float64.go rename to metrics/gauge_float64.go index 6f93920b2c..66819c9577 100644 --- a/vendor/github.com/rcrowley/go-metrics/gauge_float64.go +++ b/metrics/gauge_float64.go @@ -20,7 +20,7 @@ func GetOrRegisterGaugeFloat64(name string, r Registry) GaugeFloat64 { // NewGaugeFloat64 constructs a new StandardGaugeFloat64. func NewGaugeFloat64() GaugeFloat64 { - if UseNilMetrics { + if !Enabled { return NilGaugeFloat64{} } return &StandardGaugeFloat64{ @@ -40,7 +40,7 @@ func NewRegisteredGaugeFloat64(name string, r Registry) GaugeFloat64 { // NewFunctionalGauge constructs a new FunctionalGauge. func NewFunctionalGaugeFloat64(f func() float64) GaugeFloat64 { - if UseNilMetrics { + if !Enabled { return NilGaugeFloat64{} } return &FunctionalGaugeFloat64{value: f} diff --git a/metrics/gauge_float64_test.go b/metrics/gauge_float64_test.go new file mode 100644 index 0000000000..99e62a4030 --- /dev/null +++ b/metrics/gauge_float64_test.go @@ -0,0 +1,59 @@ +package metrics + +import "testing" + +func BenchmarkGuageFloat64(b *testing.B) { + g := NewGaugeFloat64() + b.ResetTimer() + for i := 0; i < b.N; i++ { + g.Update(float64(i)) + } +} + +func TestGaugeFloat64(t *testing.T) { + g := NewGaugeFloat64() + g.Update(float64(47.0)) + if v := g.Value(); float64(47.0) != v { + t.Errorf("g.Value(): 47.0 != %v\n", v) + } +} + +func TestGaugeFloat64Snapshot(t *testing.T) { + g := NewGaugeFloat64() + g.Update(float64(47.0)) + snapshot := g.Snapshot() + g.Update(float64(0)) + if v := snapshot.Value(); float64(47.0) != v { + t.Errorf("g.Value(): 47.0 != %v\n", v) + } +} + +func TestGetOrRegisterGaugeFloat64(t *testing.T) { + r := NewRegistry() + NewRegisteredGaugeFloat64("foo", r).Update(float64(47.0)) + t.Logf("registry: %v", r) + if g := GetOrRegisterGaugeFloat64("foo", r); float64(47.0) != g.Value() { + t.Fatal(g) + } +} + +func TestFunctionalGaugeFloat64(t *testing.T) { + var counter float64 + fg := NewFunctionalGaugeFloat64(func() float64 { + counter++ + return counter + }) + fg.Value() + fg.Value() + if counter != 2 { + t.Error("counter != 2") + } +} + +func TestGetOrRegisterFunctionalGaugeFloat64(t *testing.T) { + r := NewRegistry() + NewRegisteredFunctionalGaugeFloat64("foo", r, func() float64 { return 47 }) + if g := GetOrRegisterGaugeFloat64("foo", r); 47 != g.Value() { + t.Fatal(g) + } +} diff --git a/metrics/gauge_test.go b/metrics/gauge_test.go new file mode 100644 index 0000000000..1f2603d339 --- /dev/null +++ b/metrics/gauge_test.go @@ -0,0 +1,68 @@ +package metrics + +import ( + "fmt" + "testing" +) + +func BenchmarkGuage(b *testing.B) { + g := NewGauge() + b.ResetTimer() + for i := 0; i < b.N; i++ { + g.Update(int64(i)) + } +} + +func TestGauge(t *testing.T) { + g := NewGauge() + g.Update(int64(47)) + if v := g.Value(); 47 != v { + t.Errorf("g.Value(): 47 != %v\n", v) + } +} + +func TestGaugeSnapshot(t *testing.T) { + g := NewGauge() + g.Update(int64(47)) + snapshot := g.Snapshot() + g.Update(int64(0)) + if v := snapshot.Value(); 47 != v { + t.Errorf("g.Value(): 47 != %v\n", v) + } +} + +func TestGetOrRegisterGauge(t *testing.T) { + r := NewRegistry() + NewRegisteredGauge("foo", r).Update(47) + if g := GetOrRegisterGauge("foo", r); 47 != g.Value() { + t.Fatal(g) + } +} + +func TestFunctionalGauge(t *testing.T) { + var counter int64 + fg := NewFunctionalGauge(func() int64 { + counter++ + return counter + }) + fg.Value() + fg.Value() + if counter != 2 { + t.Error("counter != 2") + } +} + +func TestGetOrRegisterFunctionalGauge(t *testing.T) { + r := NewRegistry() + NewRegisteredFunctionalGauge("foo", r, func() int64 { return 47 }) + if g := GetOrRegisterGauge("foo", r); 47 != g.Value() { + t.Fatal(g) + } +} + +func ExampleGetOrRegisterGauge() { + m := "server.bytes_sent" + g := GetOrRegisterGauge(m, nil) + g.Update(47) + fmt.Println(g.Value()) // Output: 47 +} diff --git a/vendor/github.com/rcrowley/go-metrics/graphite.go b/metrics/graphite.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/graphite.go rename to metrics/graphite.go index abd0a7d291..142eec86be 100644 --- a/vendor/github.com/rcrowley/go-metrics/graphite.go +++ b/metrics/graphite.go @@ -39,7 +39,7 @@ func Graphite(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) { // but it takes a GraphiteConfig instead. func GraphiteWithConfig(c GraphiteConfig) { log.Printf("WARNING: This go-metrics client has been DEPRECATED! It has been moved to https://github.com/cyberdelia/go-metrics-graphite and will be removed from rcrowley/go-metrics on August 12th 2015") - for _ = range time.Tick(c.FlushInterval) { + for range time.Tick(c.FlushInterval) { if err := graphite(&c); nil != err { log.Println(err) } diff --git a/metrics/graphite_test.go b/metrics/graphite_test.go new file mode 100644 index 0000000000..c797c781df --- /dev/null +++ b/metrics/graphite_test.go @@ -0,0 +1,22 @@ +package metrics + +import ( + "net" + "time" +) + +func ExampleGraphite() { + addr, _ := net.ResolveTCPAddr("net", ":2003") + go Graphite(DefaultRegistry, 1*time.Second, "some.prefix", addr) +} + +func ExampleGraphiteWithConfig() { + addr, _ := net.ResolveTCPAddr("net", ":2003") + go GraphiteWithConfig(GraphiteConfig{ + Addr: addr, + Registry: DefaultRegistry, + FlushInterval: 1 * time.Second, + DurationUnit: time.Millisecond, + Percentiles: []float64{0.5, 0.75, 0.99, 0.999}, + }) +} diff --git a/vendor/github.com/rcrowley/go-metrics/healthcheck.go b/metrics/healthcheck.go similarity index 98% rename from vendor/github.com/rcrowley/go-metrics/healthcheck.go rename to metrics/healthcheck.go index 445131caee..f1ae31e34a 100644 --- a/vendor/github.com/rcrowley/go-metrics/healthcheck.go +++ b/metrics/healthcheck.go @@ -11,7 +11,7 @@ type Healthcheck interface { // NewHealthcheck constructs a new Healthcheck which will use the given // function to update its status. func NewHealthcheck(f func(Healthcheck)) Healthcheck { - if UseNilMetrics { + if !Enabled { return NilHealthcheck{} } return &StandardHealthcheck{nil, f} diff --git a/vendor/github.com/rcrowley/go-metrics/histogram.go b/metrics/histogram.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/histogram.go rename to metrics/histogram.go index dbc837fe4d..46f3bbd2f1 100644 --- a/vendor/github.com/rcrowley/go-metrics/histogram.go +++ b/metrics/histogram.go @@ -28,7 +28,7 @@ func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram { // NewHistogram constructs a new StandardHistogram from a Sample. func NewHistogram(s Sample) Histogram { - if UseNilMetrics { + if !Enabled { return NilHistogram{} } return &StandardHistogram{sample: s} diff --git a/metrics/histogram_test.go b/metrics/histogram_test.go new file mode 100644 index 0000000000..d7f4f0171c --- /dev/null +++ b/metrics/histogram_test.go @@ -0,0 +1,95 @@ +package metrics + +import "testing" + +func BenchmarkHistogram(b *testing.B) { + h := NewHistogram(NewUniformSample(100)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + h.Update(int64(i)) + } +} + +func TestGetOrRegisterHistogram(t *testing.T) { + r := NewRegistry() + s := NewUniformSample(100) + NewRegisteredHistogram("foo", r, s).Update(47) + if h := GetOrRegisterHistogram("foo", r, s); 1 != h.Count() { + t.Fatal(h) + } +} + +func TestHistogram10000(t *testing.T) { + h := NewHistogram(NewUniformSample(100000)) + for i := 1; i <= 10000; i++ { + h.Update(int64(i)) + } + testHistogram10000(t, h) +} + +func TestHistogramEmpty(t *testing.T) { + h := NewHistogram(NewUniformSample(100)) + if count := h.Count(); 0 != count { + t.Errorf("h.Count(): 0 != %v\n", count) + } + if min := h.Min(); 0 != min { + t.Errorf("h.Min(): 0 != %v\n", min) + } + if max := h.Max(); 0 != max { + t.Errorf("h.Max(): 0 != %v\n", max) + } + if mean := h.Mean(); 0.0 != mean { + t.Errorf("h.Mean(): 0.0 != %v\n", mean) + } + if stdDev := h.StdDev(); 0.0 != stdDev { + t.Errorf("h.StdDev(): 0.0 != %v\n", stdDev) + } + ps := h.Percentiles([]float64{0.5, 0.75, 0.99}) + if 0.0 != ps[0] { + t.Errorf("median: 0.0 != %v\n", ps[0]) + } + if 0.0 != ps[1] { + t.Errorf("75th percentile: 0.0 != %v\n", ps[1]) + } + if 0.0 != ps[2] { + t.Errorf("99th percentile: 0.0 != %v\n", ps[2]) + } +} + +func TestHistogramSnapshot(t *testing.T) { + h := NewHistogram(NewUniformSample(100000)) + for i := 1; i <= 10000; i++ { + h.Update(int64(i)) + } + snapshot := h.Snapshot() + h.Update(0) + testHistogram10000(t, snapshot) +} + +func testHistogram10000(t *testing.T, h Histogram) { + if count := h.Count(); 10000 != count { + t.Errorf("h.Count(): 10000 != %v\n", count) + } + if min := h.Min(); 1 != min { + t.Errorf("h.Min(): 1 != %v\n", min) + } + if max := h.Max(); 10000 != max { + t.Errorf("h.Max(): 10000 != %v\n", max) + } + if mean := h.Mean(); 5000.5 != mean { + t.Errorf("h.Mean(): 5000.5 != %v\n", mean) + } + if stdDev := h.StdDev(); 2886.751331514372 != stdDev { + t.Errorf("h.StdDev(): 2886.751331514372 != %v\n", stdDev) + } + ps := h.Percentiles([]float64{0.5, 0.75, 0.99}) + if 5000.5 != ps[0] { + t.Errorf("median: 5000.5 != %v\n", ps[0]) + } + if 7500.75 != ps[1] { + t.Errorf("75th percentile: 7500.75 != %v\n", ps[1]) + } + if 9900.99 != ps[2] { + t.Errorf("99th percentile: 9900.99 != %v\n", ps[2]) + } +} diff --git a/metrics/influxdb/LICENSE b/metrics/influxdb/LICENSE new file mode 100644 index 0000000000..e5bf20cdb5 --- /dev/null +++ b/metrics/influxdb/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2015 Vincent Rischmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/metrics/influxdb/README.md b/metrics/influxdb/README.md new file mode 100644 index 0000000000..b76b1a3f99 --- /dev/null +++ b/metrics/influxdb/README.md @@ -0,0 +1,30 @@ +go-metrics-influxdb +=================== + +This is a reporter for the [go-metrics](https://github.com/rcrowley/go-metrics) library which will post the metrics to [InfluxDB](https://influxdb.com/). + +Note +---- + +This is only compatible with InfluxDB 0.9+. + +Usage +----- + +```go +import "github.com/vrischmann/go-metrics-influxdb" + +go influxdb.InfluxDB( + metrics.DefaultRegistry, // metrics registry + time.Second * 10, // interval + "http://localhost:8086", // the InfluxDB url + "mydb", // your InfluxDB database + "myuser", // your InfluxDB user + "mypassword", // your InfluxDB password +) +``` + +License +------- + +go-metrics-influxdb is licensed under the MIT license. See the LICENSE file for details. diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go new file mode 100644 index 0000000000..d5cb4da66f --- /dev/null +++ b/metrics/influxdb/influxdb.go @@ -0,0 +1,227 @@ +package influxdb + +import ( + "fmt" + "log" + uurl "net/url" + "time" + + "github.com/ethereum/go-ethereum/metrics" + "github.com/influxdata/influxdb/client" +) + +type reporter struct { + reg metrics.Registry + interval time.Duration + + url uurl.URL + database string + username string + password string + namespace string + tags map[string]string + + client *client.Client + + cache map[string]int64 +} + +// InfluxDB starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval. +func InfluxDB(r metrics.Registry, d time.Duration, url, database, username, password, namespace string) { + InfluxDBWithTags(r, d, url, database, username, password, namespace, nil) +} + +// InfluxDBWithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags +func InfluxDBWithTags(r metrics.Registry, d time.Duration, url, database, username, password, namespace string, tags map[string]string) { + u, err := uurl.Parse(url) + if err != nil { + log.Printf("unable to parse InfluxDB url %s. err=%v", url, err) + return + } + + rep := &reporter{ + reg: r, + interval: d, + url: *u, + database: database, + username: username, + password: password, + namespace: namespace, + tags: tags, + cache: make(map[string]int64), + } + if err := rep.makeClient(); err != nil { + log.Printf("unable to make InfluxDB client. err=%v", err) + return + } + + rep.run() +} + +func (r *reporter) makeClient() (err error) { + r.client, err = client.NewClient(client.Config{ + URL: r.url, + Username: r.username, + Password: r.password, + }) + + return +} + +func (r *reporter) run() { + intervalTicker := time.Tick(r.interval) + pingTicker := time.Tick(time.Second * 5) + + for { + select { + case <-intervalTicker: + if err := r.send(); err != nil { + log.Printf("unable to send to InfluxDB. err=%v", err) + } + case <-pingTicker: + _, _, err := r.client.Ping() + if err != nil { + log.Printf("got error while sending a ping to InfluxDB, trying to recreate client. err=%v", err) + + if err = r.makeClient(); err != nil { + log.Printf("unable to make InfluxDB client. err=%v", err) + } + } + } + } +} + +func (r *reporter) send() error { + var pts []client.Point + + r.reg.Each(func(name string, i interface{}) { + now := time.Now() + namespace := r.namespace + + switch metric := i.(type) { + case metrics.Counter: + v := metric.Count() + l := r.cache[name] + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.count", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "value": v - l, + }, + Time: now, + }) + r.cache[name] = v + case metrics.Gauge: + ms := metric.Snapshot() + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.gauge", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "value": ms.Value(), + }, + Time: now, + }) + case metrics.GaugeFloat64: + ms := metric.Snapshot() + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.gauge", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "value": ms.Value(), + }, + Time: now, + }) + case metrics.Histogram: + ms := metric.Snapshot() + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.histogram", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "stddev": ms.StdDev(), + "variance": ms.Variance(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], + }, + Time: now, + }) + case metrics.Meter: + ms := metric.Snapshot() + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.meter", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "count": ms.Count(), + "m1": ms.Rate1(), + "m5": ms.Rate5(), + "m15": ms.Rate15(), + "mean": ms.RateMean(), + }, + Time: now, + }) + case metrics.Timer: + ms := metric.Snapshot() + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.timer", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "stddev": ms.StdDev(), + "variance": ms.Variance(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], + "m1": ms.Rate1(), + "m5": ms.Rate5(), + "m15": ms.Rate15(), + "meanrate": ms.RateMean(), + }, + Time: now, + }) + case metrics.ResettingTimer: + t := metric.Snapshot() + + if len(t.Values()) > 0 { + ps := t.Percentiles([]float64{50, 95, 99}) + val := t.Values() + pts = append(pts, client.Point{ + Measurement: fmt.Sprintf("%s%s.span", namespace, name), + Tags: r.tags, + Fields: map[string]interface{}{ + "count": len(val), + "max": val[len(val)-1], + "mean": t.Mean(), + "min": val[0], + "p50": ps[0], + "p95": ps[1], + "p99": ps[2], + }, + Time: now, + }) + } + } + }) + + bps := client.BatchPoints{ + Points: pts, + Database: r.database, + } + + _, err := r.client.Write(bps) + return err +} diff --git a/metrics/init_test.go b/metrics/init_test.go new file mode 100644 index 0000000000..43401e833c --- /dev/null +++ b/metrics/init_test.go @@ -0,0 +1,5 @@ +package metrics + +func init() { + Enabled = true +} diff --git a/metrics/json.go b/metrics/json.go new file mode 100644 index 0000000000..2087d8211e --- /dev/null +++ b/metrics/json.go @@ -0,0 +1,31 @@ +package metrics + +import ( + "encoding/json" + "io" + "time" +) + +// MarshalJSON returns a byte slice containing a JSON representation of all +// the metrics in the Registry. +func (r *StandardRegistry) MarshalJSON() ([]byte, error) { + return json.Marshal(r.GetAll()) +} + +// WriteJSON writes metrics from the given registry periodically to the +// specified io.Writer as JSON. +func WriteJSON(r Registry, d time.Duration, w io.Writer) { + for range time.Tick(d) { + WriteJSONOnce(r, w) + } +} + +// WriteJSONOnce writes metrics from the given registry to the specified +// io.Writer as JSON. +func WriteJSONOnce(r Registry, w io.Writer) { + json.NewEncoder(w).Encode(r) +} + +func (p *PrefixedRegistry) MarshalJSON() ([]byte, error) { + return json.Marshal(p.GetAll()) +} diff --git a/metrics/json_test.go b/metrics/json_test.go new file mode 100644 index 0000000000..cf70051f7a --- /dev/null +++ b/metrics/json_test.go @@ -0,0 +1,28 @@ +package metrics + +import ( + "bytes" + "encoding/json" + "testing" +) + +func TestRegistryMarshallJSON(t *testing.T) { + b := &bytes.Buffer{} + enc := json.NewEncoder(b) + r := NewRegistry() + r.Register("counter", NewCounter()) + enc.Encode(r) + if s := b.String(); "{\"counter\":{\"count\":0}}\n" != s { + t.Fatalf(s) + } +} + +func TestRegistryWriteJSONOnce(t *testing.T) { + r := NewRegistry() + r.Register("counter", NewCounter()) + b := &bytes.Buffer{} + WriteJSONOnce(r, b) + if s := b.String(); s != "{\"counter\":{\"count\":0}}\n" { + t.Fail() + } +} diff --git a/metrics/librato/client.go b/metrics/librato/client.go new file mode 100644 index 0000000000..8c0c850e38 --- /dev/null +++ b/metrics/librato/client.go @@ -0,0 +1,102 @@ +package librato + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" +) + +const Operations = "operations" +const OperationsShort = "ops" + +type LibratoClient struct { + Email, Token string +} + +// property strings +const ( + // display attributes + Color = "color" + DisplayMax = "display_max" + DisplayMin = "display_min" + DisplayUnitsLong = "display_units_long" + DisplayUnitsShort = "display_units_short" + DisplayStacked = "display_stacked" + DisplayTransform = "display_transform" + // special gauge display attributes + SummarizeFunction = "summarize_function" + Aggregate = "aggregate" + + // metric keys + Name = "name" + Period = "period" + Description = "description" + DisplayName = "display_name" + Attributes = "attributes" + + // measurement keys + MeasureTime = "measure_time" + Source = "source" + Value = "value" + + // special gauge keys + Count = "count" + Sum = "sum" + Max = "max" + Min = "min" + SumSquares = "sum_squares" + + // batch keys + Counters = "counters" + Gauges = "gauges" + + MetricsPostUrl = "https://metrics-api.librato.com/v1/metrics" +) + +type Measurement map[string]interface{} +type Metric map[string]interface{} + +type Batch struct { + Gauges []Measurement `json:"gauges,omitempty"` + Counters []Measurement `json:"counters,omitempty"` + MeasureTime int64 `json:"measure_time"` + Source string `json:"source"` +} + +func (self *LibratoClient) PostMetrics(batch Batch) (err error) { + var ( + js []byte + req *http.Request + resp *http.Response + ) + + if len(batch.Counters) == 0 && len(batch.Gauges) == 0 { + return nil + } + + if js, err = json.Marshal(batch); err != nil { + return + } + + if req, err = http.NewRequest("POST", MetricsPostUrl, bytes.NewBuffer(js)); err != nil { + return + } + + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth(self.Email, self.Token) + + if resp, err = http.DefaultClient.Do(req); err != nil { + return + } + + if resp.StatusCode != http.StatusOK { + var body []byte + if body, err = ioutil.ReadAll(resp.Body); err != nil { + body = []byte(fmt.Sprintf("(could not fetch response body for error: %s)", err)) + } + err = fmt.Errorf("Unable to post to Librato: %d %s %s", resp.StatusCode, resp.Status, string(body)) + } + return +} diff --git a/metrics/librato/librato.go b/metrics/librato/librato.go new file mode 100644 index 0000000000..f8c8c9ecb7 --- /dev/null +++ b/metrics/librato/librato.go @@ -0,0 +1,235 @@ +package librato + +import ( + "fmt" + "log" + "math" + "regexp" + "time" + + "github.com/ethereum/go-ethereum/metrics" +) + +// a regexp for extracting the unit from time.Duration.String +var unitRegexp = regexp.MustCompile(`[^\\d]+$`) + +// a helper that turns a time.Duration into librato display attributes for timer metrics +func translateTimerAttributes(d time.Duration) (attrs map[string]interface{}) { + attrs = make(map[string]interface{}) + attrs[DisplayTransform] = fmt.Sprintf("x/%d", int64(d)) + attrs[DisplayUnitsShort] = string(unitRegexp.Find([]byte(d.String()))) + return +} + +type Reporter struct { + Email, Token string + Namespace string + Source string + Interval time.Duration + Registry metrics.Registry + Percentiles []float64 // percentiles to report on histogram metrics + TimerAttributes map[string]interface{} // units in which timers will be displayed + intervalSec int64 +} + +func NewReporter(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) *Reporter { + return &Reporter{e, t, "", s, d, r, p, translateTimerAttributes(u), int64(d / time.Second)} +} + +func Librato(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) { + NewReporter(r, d, e, t, s, p, u).Run() +} + +func (self *Reporter) Run() { + log.Printf("WARNING: This client has been DEPRECATED! It has been moved to https://github.com/mihasya/go-metrics-librato and will be removed from rcrowley/go-metrics on August 5th 2015") + ticker := time.Tick(self.Interval) + metricsApi := &LibratoClient{self.Email, self.Token} + for now := range ticker { + var metrics Batch + var err error + if metrics, err = self.BuildRequest(now, self.Registry); err != nil { + log.Printf("ERROR constructing librato request body %s", err) + continue + } + if err := metricsApi.PostMetrics(metrics); err != nil { + log.Printf("ERROR sending metrics to librato %s", err) + continue + } + } +} + +// calculate sum of squares from data provided by metrics.Histogram +// see http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods +func sumSquares(s metrics.Sample) float64 { + count := float64(s.Count()) + sumSquared := math.Pow(count*s.Mean(), 2) + sumSquares := math.Pow(count*s.StdDev(), 2) + sumSquared/count + if math.IsNaN(sumSquares) { + return 0.0 + } + return sumSquares +} +func sumSquaresTimer(t metrics.Timer) float64 { + count := float64(t.Count()) + sumSquared := math.Pow(count*t.Mean(), 2) + sumSquares := math.Pow(count*t.StdDev(), 2) + sumSquared/count + if math.IsNaN(sumSquares) { + return 0.0 + } + return sumSquares +} + +func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Batch, err error) { + snapshot = Batch{ + // coerce timestamps to a stepping fn so that they line up in Librato graphs + MeasureTime: (now.Unix() / self.intervalSec) * self.intervalSec, + Source: self.Source, + } + snapshot.Gauges = make([]Measurement, 0) + snapshot.Counters = make([]Measurement, 0) + histogramGaugeCount := 1 + len(self.Percentiles) + r.Each(func(name string, metric interface{}) { + if self.Namespace != "" { + name = fmt.Sprintf("%s.%s", self.Namespace, name) + } + measurement := Measurement{} + measurement[Period] = self.Interval.Seconds() + switch m := metric.(type) { + case metrics.Counter: + if m.Count() > 0 { + measurement[Name] = fmt.Sprintf("%s.%s", name, "count") + measurement[Value] = float64(m.Count()) + measurement[Attributes] = map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + } + snapshot.Counters = append(snapshot.Counters, measurement) + } + case metrics.Gauge: + measurement[Name] = name + measurement[Value] = float64(m.Value()) + snapshot.Gauges = append(snapshot.Gauges, measurement) + case metrics.GaugeFloat64: + measurement[Name] = name + measurement[Value] = m.Value() + snapshot.Gauges = append(snapshot.Gauges, measurement) + case metrics.Histogram: + if m.Count() > 0 { + gauges := make([]Measurement, histogramGaugeCount) + s := m.Sample() + measurement[Name] = fmt.Sprintf("%s.%s", name, "hist") + measurement[Count] = uint64(s.Count()) + measurement[Max] = float64(s.Max()) + measurement[Min] = float64(s.Min()) + measurement[Sum] = float64(s.Sum()) + measurement[SumSquares] = sumSquares(s) + gauges[0] = measurement + for i, p := range self.Percentiles { + gauges[i+1] = Measurement{ + Name: fmt.Sprintf("%s.%.2f", measurement[Name], p), + Value: s.Percentile(p), + Period: measurement[Period], + } + } + snapshot.Gauges = append(snapshot.Gauges, gauges...) + } + case metrics.Meter: + measurement[Name] = name + measurement[Value] = float64(m.Count()) + snapshot.Counters = append(snapshot.Counters, measurement) + snapshot.Gauges = append(snapshot.Gauges, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "1min"), + Value: m.Rate1(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "5min"), + Value: m.Rate5(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "15min"), + Value: m.Rate15(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + ) + case metrics.Timer: + measurement[Name] = name + measurement[Value] = float64(m.Count()) + snapshot.Counters = append(snapshot.Counters, measurement) + if m.Count() > 0 { + libratoName := fmt.Sprintf("%s.%s", name, "timer.mean") + gauges := make([]Measurement, histogramGaugeCount) + gauges[0] = Measurement{ + Name: libratoName, + Count: uint64(m.Count()), + Sum: m.Mean() * float64(m.Count()), + Max: float64(m.Max()), + Min: float64(m.Min()), + SumSquares: sumSquaresTimer(m), + Period: int64(self.Interval.Seconds()), + Attributes: self.TimerAttributes, + } + for i, p := range self.Percentiles { + gauges[i+1] = Measurement{ + Name: fmt.Sprintf("%s.timer.%2.0f", name, p*100), + Value: m.Percentile(p), + Period: int64(self.Interval.Seconds()), + Attributes: self.TimerAttributes, + } + } + snapshot.Gauges = append(snapshot.Gauges, gauges...) + snapshot.Gauges = append(snapshot.Gauges, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "rate.1min"), + Value: m.Rate1(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "rate.5min"), + Value: m.Rate5(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + Measurement{ + Name: fmt.Sprintf("%s.%s", name, "rate.15min"), + Value: m.Rate15(), + Period: int64(self.Interval.Seconds()), + Attributes: map[string]interface{}{ + DisplayUnitsLong: Operations, + DisplayUnitsShort: OperationsShort, + DisplayMin: "0", + }, + }, + ) + } + } + }) + return +} diff --git a/vendor/github.com/rcrowley/go-metrics/log.go b/metrics/log.go similarity index 98% rename from vendor/github.com/rcrowley/go-metrics/log.go rename to metrics/log.go index f8074c0457..0c8ea7c971 100644 --- a/vendor/github.com/rcrowley/go-metrics/log.go +++ b/metrics/log.go @@ -18,7 +18,7 @@ func LogScaled(r Registry, freq time.Duration, scale time.Duration, l Logger) { du := float64(scale) duSuffix := scale.String()[1:] - for _ = range time.Tick(freq) { + for range time.Tick(freq) { r.Each(func(name string, i interface{}) { switch metric := i.(type) { case Counter: diff --git a/vendor/github.com/rcrowley/go-metrics/memory.md b/metrics/memory.md similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/memory.md rename to metrics/memory.md diff --git a/vendor/github.com/rcrowley/go-metrics/meter.go b/metrics/meter.go similarity index 81% rename from vendor/github.com/rcrowley/go-metrics/meter.go rename to metrics/meter.go index 0389ab0b8f..82b2141a62 100644 --- a/vendor/github.com/rcrowley/go-metrics/meter.go +++ b/metrics/meter.go @@ -15,10 +15,13 @@ type Meter interface { Rate15() float64 RateMean() float64 Snapshot() Meter + Stop() } // GetOrRegisterMeter returns an existing Meter or constructs and registers a // new StandardMeter. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. func GetOrRegisterMeter(name string, r Registry) Meter { if nil == r { r = DefaultRegistry @@ -27,14 +30,15 @@ func GetOrRegisterMeter(name string, r Registry) Meter { } // NewMeter constructs a new StandardMeter and launches a goroutine. +// Be sure to call Stop() once the meter is of no use to allow for garbage collection. func NewMeter() Meter { - if UseNilMetrics { + if !Enabled { return NilMeter{} } m := newStandardMeter() arbiter.Lock() defer arbiter.Unlock() - arbiter.meters = append(arbiter.meters, m) + arbiter.meters[m] = struct{}{} if !arbiter.started { arbiter.started = true go arbiter.tick() @@ -44,6 +48,8 @@ func NewMeter() Meter { // NewMeter constructs and registers a new StandardMeter and launches a // goroutine. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. func NewRegisteredMeter(name string, r Registry) Meter { c := NewMeter() if nil == r { @@ -86,6 +92,9 @@ func (m *MeterSnapshot) RateMean() float64 { return m.rateMean } // Snapshot returns the snapshot. func (m *MeterSnapshot) Snapshot() Meter { return m } +// Stop is a no-op. +func (m *MeterSnapshot) Stop() {} + // NilMeter is a no-op Meter. type NilMeter struct{} @@ -110,12 +119,16 @@ func (NilMeter) RateMean() float64 { return 0.0 } // Snapshot is a no-op. func (NilMeter) Snapshot() Meter { return NilMeter{} } +// Stop is a no-op. +func (NilMeter) Stop() {} + // StandardMeter is the standard implementation of a Meter. type StandardMeter struct { lock sync.RWMutex snapshot *MeterSnapshot a1, a5, a15 EWMA startTime time.Time + stopped bool } func newStandardMeter() *StandardMeter { @@ -128,6 +141,19 @@ func newStandardMeter() *StandardMeter { } } +// Stop stops the meter, Mark() will be a no-op if you use it after being stopped. +func (m *StandardMeter) Stop() { + m.lock.Lock() + stopped := m.stopped + m.stopped = true + m.lock.Unlock() + if !stopped { + arbiter.Lock() + delete(arbiter.meters, m) + arbiter.Unlock() + } +} + // Count returns the number of events recorded. func (m *StandardMeter) Count() int64 { m.lock.RLock() @@ -136,10 +162,13 @@ func (m *StandardMeter) Count() int64 { return count } -// Mark records the occurance of n events. +// Mark records the occurrence of n events. func (m *StandardMeter) Mark(n int64) { m.lock.Lock() defer m.lock.Unlock() + if m.stopped { + return + } m.snapshot.count += n m.a1.Update(n) m.a5.Update(n) @@ -205,29 +234,28 @@ func (m *StandardMeter) tick() { m.updateSnapshot() } +// meterArbiter ticks meters every 5s from a single goroutine. +// meters are references in a set for future stopping. type meterArbiter struct { sync.RWMutex started bool - meters []*StandardMeter + meters map[*StandardMeter]struct{} ticker *time.Ticker } -var arbiter = meterArbiter{ticker: time.NewTicker(5e9)} +var arbiter = meterArbiter{ticker: time.NewTicker(5e9), meters: make(map[*StandardMeter]struct{})} // Ticks meters on the scheduled interval func (ma *meterArbiter) tick() { - for { - select { - case <-ma.ticker.C: - ma.tickMeters() - } + for range ma.ticker.C { + ma.tickMeters() } } func (ma *meterArbiter) tickMeters() { ma.RLock() defer ma.RUnlock() - for _, meter := range ma.meters { + for meter := range ma.meters { meter.tick() } } diff --git a/metrics/meter_test.go b/metrics/meter_test.go new file mode 100644 index 0000000000..e889222601 --- /dev/null +++ b/metrics/meter_test.go @@ -0,0 +1,73 @@ +package metrics + +import ( + "testing" + "time" +) + +func BenchmarkMeter(b *testing.B) { + m := NewMeter() + b.ResetTimer() + for i := 0; i < b.N; i++ { + m.Mark(1) + } +} + +func TestGetOrRegisterMeter(t *testing.T) { + r := NewRegistry() + NewRegisteredMeter("foo", r).Mark(47) + if m := GetOrRegisterMeter("foo", r); 47 != m.Count() { + t.Fatal(m) + } +} + +func TestMeterDecay(t *testing.T) { + ma := meterArbiter{ + ticker: time.NewTicker(time.Millisecond), + meters: make(map[*StandardMeter]struct{}), + } + m := newStandardMeter() + ma.meters[m] = struct{}{} + go ma.tick() + m.Mark(1) + rateMean := m.RateMean() + time.Sleep(100 * time.Millisecond) + if m.RateMean() >= rateMean { + t.Error("m.RateMean() didn't decrease") + } +} + +func TestMeterNonzero(t *testing.T) { + m := NewMeter() + m.Mark(3) + if count := m.Count(); 3 != count { + t.Errorf("m.Count(): 3 != %v\n", count) + } +} + +func TestMeterStop(t *testing.T) { + l := len(arbiter.meters) + m := NewMeter() + if len(arbiter.meters) != l+1 { + t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters)) + } + m.Stop() + if len(arbiter.meters) != l { + t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters)) + } +} + +func TestMeterSnapshot(t *testing.T) { + m := NewMeter() + m.Mark(1) + if snapshot := m.Snapshot(); m.RateMean() != snapshot.RateMean() { + t.Fatal(snapshot) + } +} + +func TestMeterZero(t *testing.T) { + m := NewMeter() + if count := m.Count(); 0 != count { + t.Errorf("m.Count(): 0 != %v\n", count) + } +} diff --git a/metrics/metrics.go b/metrics/metrics.go index c82661d802..e24324814c 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,20 +1,8 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. +// Go port of Coda Hale's Metrics library // -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. +//
// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see
. - -// Package metrics provides general system and process level metrics collection. +// Coda Hale's original work:
package metrics import ( @@ -24,17 +12,19 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - "github.com/rcrowley/go-metrics" - "github.com/rcrowley/go-metrics/exp" ) +// Enabled is checked by the constructor functions for all of the +// standard metrics. If it is true, the metric returned is a stub. +// +// This global kill-switch helps quantify the observer effect and makes +// for less cluttered pprof profiles. +var Enabled bool = false + // MetricsEnabledFlag is the CLI flag name to use to enable metrics collections. const MetricsEnabledFlag = "metrics" const DashboardEnabledFlag = "dashboard" -// Enabled is the flag specifying if metrics are enable or not. -var Enabled = false - // Init enables or disables the metrics system. Since we need this to run before // any other code gets to create meters and timers, we'll actually do an ugly hack // and peek into the command line args for the metrics flag. @@ -45,34 +35,6 @@ func init() { Enabled = true } } - exp.Exp(metrics.DefaultRegistry) -} - -// NewCounter create a new metrics Counter, either a real one of a NOP stub depending -// on the metrics flag. -func NewCounter(name string) metrics.Counter { - if !Enabled { - return new(metrics.NilCounter) - } - return metrics.GetOrRegisterCounter(name, metrics.DefaultRegistry) -} - -// NewMeter create a new metrics Meter, either a real one of a NOP stub depending -// on the metrics flag. -func NewMeter(name string) metrics.Meter { - if !Enabled { - return new(metrics.NilMeter) - } - return metrics.GetOrRegisterMeter(name, metrics.DefaultRegistry) -} - -// NewTimer create a new metrics Timer, either a real one of a NOP stub depending -// on the metrics flag. -func NewTimer(name string) metrics.Timer { - if !Enabled { - return new(metrics.NilTimer) - } - return metrics.GetOrRegisterTimer(name, metrics.DefaultRegistry) } // CollectProcessMetrics periodically collects various metrics about the running @@ -90,17 +52,17 @@ func CollectProcessMetrics(refresh time.Duration) { diskstats[i] = new(DiskStats) } // Define the various metrics to collect - memAllocs := metrics.GetOrRegisterMeter("system/memory/allocs", metrics.DefaultRegistry) - memFrees := metrics.GetOrRegisterMeter("system/memory/frees", metrics.DefaultRegistry) - memInuse := metrics.GetOrRegisterMeter("system/memory/inuse", metrics.DefaultRegistry) - memPauses := metrics.GetOrRegisterMeter("system/memory/pauses", metrics.DefaultRegistry) + memAllocs := GetOrRegisterMeter("system/memory/allocs", DefaultRegistry) + memFrees := GetOrRegisterMeter("system/memory/frees", DefaultRegistry) + memInuse := GetOrRegisterMeter("system/memory/inuse", DefaultRegistry) + memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry) - var diskReads, diskReadBytes, diskWrites, diskWriteBytes metrics.Meter + var diskReads, diskReadBytes, diskWrites, diskWriteBytes Meter if err := ReadDiskStats(diskstats[0]); err == nil { - diskReads = metrics.GetOrRegisterMeter("system/disk/readcount", metrics.DefaultRegistry) - diskReadBytes = metrics.GetOrRegisterMeter("system/disk/readdata", metrics.DefaultRegistry) - diskWrites = metrics.GetOrRegisterMeter("system/disk/writecount", metrics.DefaultRegistry) - diskWriteBytes = metrics.GetOrRegisterMeter("system/disk/writedata", metrics.DefaultRegistry) + diskReads = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry) + diskReadBytes = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry) + diskWrites = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry) + diskWriteBytes = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry) } else { log.Debug("Failed to read disk metrics", "err", err) } diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go new file mode 100644 index 0000000000..726fba3475 --- /dev/null +++ b/metrics/metrics_test.go @@ -0,0 +1,124 @@ +package metrics + +import ( + "fmt" + "io/ioutil" + "log" + "sync" + "testing" +) + +const FANOUT = 128 + +// Stop the compiler from complaining during debugging. +var ( + _ = ioutil.Discard + _ = log.LstdFlags +) + +func BenchmarkMetrics(b *testing.B) { + r := NewRegistry() + c := NewRegisteredCounter("counter", r) + g := NewRegisteredGauge("gauge", r) + gf := NewRegisteredGaugeFloat64("gaugefloat64", r) + h := NewRegisteredHistogram("histogram", r, NewUniformSample(100)) + m := NewRegisteredMeter("meter", r) + t := NewRegisteredTimer("timer", r) + RegisterDebugGCStats(r) + RegisterRuntimeMemStats(r) + b.ResetTimer() + ch := make(chan bool) + + wgD := &sync.WaitGroup{} + /* + wgD.Add(1) + go func() { + defer wgD.Done() + //log.Println("go CaptureDebugGCStats") + for { + select { + case <-ch: + //log.Println("done CaptureDebugGCStats") + return + default: + CaptureDebugGCStatsOnce(r) + } + } + }() + //*/ + + wgR := &sync.WaitGroup{} + //* + wgR.Add(1) + go func() { + defer wgR.Done() + //log.Println("go CaptureRuntimeMemStats") + for { + select { + case <-ch: + //log.Println("done CaptureRuntimeMemStats") + return + default: + CaptureRuntimeMemStatsOnce(r) + } + } + }() + //*/ + + wgW := &sync.WaitGroup{} + /* + wgW.Add(1) + go func() { + defer wgW.Done() + //log.Println("go Write") + for { + select { + case <-ch: + //log.Println("done Write") + return + default: + WriteOnce(r, ioutil.Discard) + } + } + }() + //*/ + + wg := &sync.WaitGroup{} + wg.Add(FANOUT) + for i := 0; i < FANOUT; i++ { + go func(i int) { + defer wg.Done() + //log.Println("go", i) + for i := 0; i < b.N; i++ { + c.Inc(1) + g.Update(int64(i)) + gf.Update(float64(i)) + h.Update(int64(i)) + m.Mark(1) + t.Update(1) + } + //log.Println("done", i) + }(i) + } + wg.Wait() + close(ch) + wgD.Wait() + wgR.Wait() + wgW.Wait() +} + +func Example() { + c := NewCounter() + Register("money", c) + c.Inc(17) + + // Threadsafe registration + t := GetOrRegisterTimer("db.get.latency", nil) + t.Time(func() {}) + t.Update(1) + + fmt.Println(c.Count()) + fmt.Println(t.Min()) + // Output: 17 + // 1 +} diff --git a/vendor/github.com/rcrowley/go-metrics/opentsdb.go b/metrics/opentsdb.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/opentsdb.go rename to metrics/opentsdb.go index 266b6c93d2..df7f152ed2 100644 --- a/vendor/github.com/rcrowley/go-metrics/opentsdb.go +++ b/metrics/opentsdb.go @@ -38,7 +38,7 @@ func OpenTSDB(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) { // OpenTSDBWithConfig is a blocking exporter function just like OpenTSDB, // but it takes a OpenTSDBConfig instead. func OpenTSDBWithConfig(c OpenTSDBConfig) { - for _ = range time.Tick(c.FlushInterval) { + for range time.Tick(c.FlushInterval) { if err := openTSDB(&c); nil != err { log.Println(err) } diff --git a/metrics/opentsdb_test.go b/metrics/opentsdb_test.go new file mode 100644 index 0000000000..c43728960e --- /dev/null +++ b/metrics/opentsdb_test.go @@ -0,0 +1,21 @@ +package metrics + +import ( + "net" + "time" +) + +func ExampleOpenTSDB() { + addr, _ := net.ResolveTCPAddr("net", ":2003") + go OpenTSDB(DefaultRegistry, 1*time.Second, "some.prefix", addr) +} + +func ExampleOpenTSDBWithConfig() { + addr, _ := net.ResolveTCPAddr("net", ":2003") + go OpenTSDBWithConfig(OpenTSDBConfig{ + Addr: addr, + Registry: DefaultRegistry, + FlushInterval: 1 * time.Second, + DurationUnit: time.Millisecond, + }) +} diff --git a/vendor/github.com/rcrowley/go-metrics/registry.go b/metrics/registry.go similarity index 76% rename from vendor/github.com/rcrowley/go-metrics/registry.go rename to metrics/registry.go index 2bb7a1e7d0..cc34c9dfd2 100644 --- a/vendor/github.com/rcrowley/go-metrics/registry.go +++ b/metrics/registry.go @@ -29,6 +29,9 @@ type Registry interface { // Get the metric by the given name or nil if none is registered. Get(string) interface{} + // GetAll metrics in the Registry. + GetAll() map[string]map[string]interface{} + // Gets an existing metric or registers the given one. // The interface can be the metric to register if not found in registry, // or a function returning the metric for lazy instantiation. @@ -109,10 +112,72 @@ func (r *StandardRegistry) RunHealthchecks() { } } +// GetAll metrics in the Registry +func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { + data := make(map[string]map[string]interface{}) + r.Each(func(name string, i interface{}) { + values := make(map[string]interface{}) + switch metric := i.(type) { + case Counter: + values["count"] = metric.Count() + case Gauge: + values["value"] = metric.Value() + case GaugeFloat64: + values["value"] = metric.Value() + case Healthcheck: + values["error"] = nil + metric.Check() + if err := metric.Error(); nil != err { + values["error"] = metric.Error().Error() + } + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = h.Count() + values["min"] = h.Min() + values["max"] = h.Max() + values["mean"] = h.Mean() + values["stddev"] = h.StdDev() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] + case Meter: + m := metric.Snapshot() + values["count"] = m.Count() + values["1m.rate"] = m.Rate1() + values["5m.rate"] = m.Rate5() + values["15m.rate"] = m.Rate15() + values["mean.rate"] = m.RateMean() + case Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = t.Count() + values["min"] = t.Min() + values["max"] = t.Max() + values["mean"] = t.Mean() + values["stddev"] = t.StdDev() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] + values["1m.rate"] = t.Rate1() + values["5m.rate"] = t.Rate5() + values["15m.rate"] = t.Rate15() + values["mean.rate"] = t.RateMean() + } + data[name] = values + }) + return data +} + // Unregister the metric with the given name. func (r *StandardRegistry) Unregister(name string) { r.mutex.Lock() defer r.mutex.Unlock() + r.stop(name) delete(r.metrics, name) } @@ -120,7 +185,8 @@ func (r *StandardRegistry) Unregister(name string) { func (r *StandardRegistry) UnregisterAll() { r.mutex.Lock() defer r.mutex.Unlock() - for name, _ := range r.metrics { + for name := range r.metrics { + r.stop(name) delete(r.metrics, name) } } @@ -130,7 +196,7 @@ func (r *StandardRegistry) register(name string, i interface{}) error { return DuplicateMetric(name) } switch i.(type) { - case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer: + case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer, ResettingTimer: r.metrics[name] = i } return nil @@ -146,6 +212,19 @@ func (r *StandardRegistry) registered() map[string]interface{} { return metrics } +func (r *StandardRegistry) stop(name string) { + if i, ok := r.metrics[name]; ok { + if s, ok := i.(Stoppable); ok { + s.Stop() + } + } +} + +// Stoppable defines the metrics which has to be stopped. +type Stoppable interface { + Stop() +} + type PrefixedRegistry struct { underlying Registry prefix string @@ -216,6 +295,11 @@ func (r *PrefixedRegistry) RunHealthchecks() { r.underlying.RunHealthchecks() } +// GetAll metrics in the Registry +func (r *PrefixedRegistry) GetAll() map[string]map[string]interface{} { + return r.underlying.GetAll() +} + // Unregister the metric with the given name. The name will be prefixed. func (r *PrefixedRegistry) Unregister(name string) { realName := r.prefix + name diff --git a/metrics/registry_test.go b/metrics/registry_test.go new file mode 100644 index 0000000000..a63e485fe9 --- /dev/null +++ b/metrics/registry_test.go @@ -0,0 +1,305 @@ +package metrics + +import ( + "testing" +) + +func BenchmarkRegistry(b *testing.B) { + r := NewRegistry() + r.Register("foo", NewCounter()) + b.ResetTimer() + for i := 0; i < b.N; i++ { + r.Each(func(string, interface{}) {}) + } +} + +func TestRegistry(t *testing.T) { + r := NewRegistry() + r.Register("foo", NewCounter()) + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if "foo" != name { + t.Fatal(name) + } + if _, ok := iface.(Counter); !ok { + t.Fatal(iface) + } + }) + if 1 != i { + t.Fatal(i) + } + r.Unregister("foo") + i = 0 + r.Each(func(string, interface{}) { i++ }) + if 0 != i { + t.Fatal(i) + } +} + +func TestRegistryDuplicate(t *testing.T) { + r := NewRegistry() + if err := r.Register("foo", NewCounter()); nil != err { + t.Fatal(err) + } + if err := r.Register("foo", NewGauge()); nil == err { + t.Fatal(err) + } + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if _, ok := iface.(Counter); !ok { + t.Fatal(iface) + } + }) + if 1 != i { + t.Fatal(i) + } +} + +func TestRegistryGet(t *testing.T) { + r := NewRegistry() + r.Register("foo", NewCounter()) + if count := r.Get("foo").(Counter).Count(); 0 != count { + t.Fatal(count) + } + r.Get("foo").(Counter).Inc(1) + if count := r.Get("foo").(Counter).Count(); 1 != count { + t.Fatal(count) + } +} + +func TestRegistryGetOrRegister(t *testing.T) { + r := NewRegistry() + + // First metric wins with GetOrRegister + _ = r.GetOrRegister("foo", NewCounter()) + m := r.GetOrRegister("foo", NewGauge()) + if _, ok := m.(Counter); !ok { + t.Fatal(m) + } + + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if name != "foo" { + t.Fatal(name) + } + if _, ok := iface.(Counter); !ok { + t.Fatal(iface) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestRegistryGetOrRegisterWithLazyInstantiation(t *testing.T) { + r := NewRegistry() + + // First metric wins with GetOrRegister + _ = r.GetOrRegister("foo", NewCounter) + m := r.GetOrRegister("foo", NewGauge) + if _, ok := m.(Counter); !ok { + t.Fatal(m) + } + + i := 0 + r.Each(func(name string, iface interface{}) { + i++ + if name != "foo" { + t.Fatal(name) + } + if _, ok := iface.(Counter); !ok { + t.Fatal(iface) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestRegistryUnregister(t *testing.T) { + l := len(arbiter.meters) + r := NewRegistry() + r.Register("foo", NewCounter()) + r.Register("bar", NewMeter()) + r.Register("baz", NewTimer()) + if len(arbiter.meters) != l+2 { + t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters)) + } + r.Unregister("foo") + r.Unregister("bar") + r.Unregister("baz") + if len(arbiter.meters) != l { + t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters)) + } +} + +func TestPrefixedChildRegistryGetOrRegister(t *testing.T) { + r := NewRegistry() + pr := NewPrefixedChildRegistry(r, "prefix.") + + _ = pr.GetOrRegister("foo", NewCounter()) + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryGetOrRegister(t *testing.T) { + r := NewPrefixedRegistry("prefix.") + + _ = r.GetOrRegister("foo", NewCounter()) + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryRegister(t *testing.T) { + r := NewPrefixedRegistry("prefix.") + err := r.Register("foo", NewCounter()) + c := NewCounter() + Register("bar", c) + if err != nil { + t.Fatal(err.Error()) + } + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryUnregister(t *testing.T) { + r := NewPrefixedRegistry("prefix.") + + _ = r.Register("foo", NewCounter()) + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } + + r.Unregister("foo") + + i = 0 + r.Each(func(name string, m interface{}) { + i++ + }) + + if i != 0 { + t.Fatal(i) + } +} + +func TestPrefixedRegistryGet(t *testing.T) { + pr := NewPrefixedRegistry("prefix.") + name := "foo" + pr.Register(name, NewCounter()) + + fooCounter := pr.Get(name) + if fooCounter == nil { + t.Fatal(name) + } +} + +func TestPrefixedChildRegistryGet(t *testing.T) { + r := NewRegistry() + pr := NewPrefixedChildRegistry(r, "prefix.") + name := "foo" + pr.Register(name, NewCounter()) + fooCounter := pr.Get(name) + if fooCounter == nil { + t.Fatal(name) + } +} + +func TestChildPrefixedRegistryRegister(t *testing.T) { + r := NewPrefixedChildRegistry(DefaultRegistry, "prefix.") + err := r.Register("foo", NewCounter()) + c := NewCounter() + Register("bar", c) + if err != nil { + t.Fatal(err.Error()) + } + + i := 0 + r.Each(func(name string, m interface{}) { + i++ + if name != "prefix.foo" { + t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestChildPrefixedRegistryOfChildRegister(t *testing.T) { + r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") + r2 := NewPrefixedChildRegistry(r, "prefix2.") + err := r.Register("foo2", NewCounter()) + if err != nil { + t.Fatal(err.Error()) + } + err = r2.Register("baz", NewCounter()) + c := NewCounter() + Register("bars", c) + + i := 0 + r2.Each(func(name string, m interface{}) { + i++ + if name != "prefix.prefix2.baz" { + //t.Fatal(name) + } + }) + if i != 1 { + t.Fatal(i) + } +} + +func TestWalkRegistries(t *testing.T) { + r := NewPrefixedChildRegistry(NewRegistry(), "prefix.") + r2 := NewPrefixedChildRegistry(r, "prefix2.") + err := r.Register("foo2", NewCounter()) + if err != nil { + t.Fatal(err.Error()) + } + err = r2.Register("baz", NewCounter()) + c := NewCounter() + Register("bars", c) + + _, prefix := findPrefix(r2, "") + if "prefix.prefix2." != prefix { + t.Fatal(prefix) + } + +} diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go new file mode 100644 index 0000000000..57bcb31343 --- /dev/null +++ b/metrics/resetting_timer.go @@ -0,0 +1,237 @@ +package metrics + +import ( + "math" + "sort" + "sync" + "time" +) + +// Initial slice capacity for the values stored in a ResettingTimer +const InitialResettingTimerSliceCap = 10 + +// ResettingTimer is used for storing aggregated values for timers, which are reset on every flush interval. +type ResettingTimer interface { + Values() []int64 + Snapshot() ResettingTimer + Percentiles([]float64) []int64 + Mean() float64 + Time(func()) + Update(time.Duration) + UpdateSince(time.Time) +} + +// GetOrRegisterResettingTimer returns an existing ResettingTimer or constructs and registers a +// new StandardResettingTimer. +func GetOrRegisterResettingTimer(name string, r Registry) ResettingTimer { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewResettingTimer).(ResettingTimer) +} + +// NewRegisteredResettingTimer constructs and registers a new StandardResettingTimer. +func NewRegisteredResettingTimer(name string, r Registry) ResettingTimer { + c := NewResettingTimer() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// NewResettingTimer constructs a new StandardResettingTimer +func NewResettingTimer() ResettingTimer { + if !Enabled { + return NilResettingTimer{} + } + return &StandardResettingTimer{ + values: make([]int64, 0, InitialResettingTimerSliceCap), + } +} + +// NilResettingTimer is a no-op ResettingTimer. +type NilResettingTimer struct { +} + +// Values is a no-op. +func (NilResettingTimer) Values() []int64 { return nil } + +// Snapshot is a no-op. +func (NilResettingTimer) Snapshot() ResettingTimer { return NilResettingTimer{} } + +// Time is a no-op. +func (NilResettingTimer) Time(func()) {} + +// Update is a no-op. +func (NilResettingTimer) Update(time.Duration) {} + +// Percentiles panics. +func (NilResettingTimer) Percentiles([]float64) []int64 { + panic("Percentiles called on a NilResettingTimer") +} + +// Mean panics. +func (NilResettingTimer) Mean() float64 { + panic("Mean called on a NilResettingTimer") +} + +// UpdateSince is a no-op. +func (NilResettingTimer) UpdateSince(time.Time) {} + +// StandardResettingTimer is the standard implementation of a ResettingTimer. +// and Meter. +type StandardResettingTimer struct { + values []int64 + mutex sync.Mutex +} + +// Values returns a slice with all measurements. +func (t *StandardResettingTimer) Values() []int64 { + return t.values +} + +// Snapshot resets the timer and returns a read-only copy of its contents. +func (t *StandardResettingTimer) Snapshot() ResettingTimer { + t.mutex.Lock() + defer t.mutex.Unlock() + currentValues := t.values + t.values = make([]int64, 0, InitialResettingTimerSliceCap) + + return &ResettingTimerSnapshot{ + values: currentValues, + } +} + +// Percentiles panics. +func (t *StandardResettingTimer) Percentiles([]float64) []int64 { + panic("Percentiles called on a StandardResettingTimer") +} + +// Mean panics. +func (t *StandardResettingTimer) Mean() float64 { + panic("Mean called on a StandardResettingTimer") +} + +// Record the duration of the execution of the given function. +func (t *StandardResettingTimer) Time(f func()) { + ts := time.Now() + f() + t.Update(time.Since(ts)) +} + +// Record the duration of an event. +func (t *StandardResettingTimer) Update(d time.Duration) { + t.mutex.Lock() + defer t.mutex.Unlock() + t.values = append(t.values, int64(d)) +} + +// Record the duration of an event that started at a time and ends now. +func (t *StandardResettingTimer) UpdateSince(ts time.Time) { + t.mutex.Lock() + defer t.mutex.Unlock() + t.values = append(t.values, int64(time.Since(ts))) +} + +// ResettingTimerSnapshot is a point-in-time copy of another ResettingTimer. +type ResettingTimerSnapshot struct { + values []int64 + mean float64 + thresholdBoundaries []int64 + calculated bool +} + +// Snapshot returns the snapshot. +func (t *ResettingTimerSnapshot) Snapshot() ResettingTimer { return t } + +// Time panics. +func (*ResettingTimerSnapshot) Time(func()) { + panic("Time called on a ResettingTimerSnapshot") +} + +// Update panics. +func (*ResettingTimerSnapshot) Update(time.Duration) { + panic("Update called on a ResettingTimerSnapshot") +} + +// UpdateSince panics. +func (*ResettingTimerSnapshot) UpdateSince(time.Time) { + panic("UpdateSince called on a ResettingTimerSnapshot") +} + +// Values returns all values from snapshot. +func (t *ResettingTimerSnapshot) Values() []int64 { + return t.values +} + +// Percentiles returns the boundaries for the input percentiles. +func (t *ResettingTimerSnapshot) Percentiles(percentiles []float64) []int64 { + t.calc(percentiles) + + return t.thresholdBoundaries +} + +// Mean returns the mean of the snapshotted values +func (t *ResettingTimerSnapshot) Mean() float64 { + if !t.calculated { + t.calc([]float64{}) + } + + return t.mean +} + +func (t *ResettingTimerSnapshot) calc(percentiles []float64) { + sort.Sort(Int64Slice(t.values)) + + count := len(t.values) + if count > 0 { + min := t.values[0] + max := t.values[count-1] + + cumulativeValues := make([]int64, count) + cumulativeValues[0] = min + for i := 1; i < count; i++ { + cumulativeValues[i] = t.values[i] + cumulativeValues[i-1] + } + + t.thresholdBoundaries = make([]int64, len(percentiles)) + + thresholdBoundary := max + + for i, pct := range percentiles { + if count > 1 { + var abs float64 + if pct >= 0 { + abs = pct + } else { + abs = 100 + pct + } + // poor man's math.Round(x): + // math.Floor(x + 0.5) + indexOfPerc := int(math.Floor(((abs / 100.0) * float64(count)) + 0.5)) + if pct >= 0 { + indexOfPerc -= 1 // index offset=0 + } + thresholdBoundary = t.values[indexOfPerc] + } + + t.thresholdBoundaries[i] = thresholdBoundary + } + + sum := cumulativeValues[count-1] + t.mean = float64(sum) / float64(count) + } else { + t.thresholdBoundaries = make([]int64, len(percentiles)) + t.mean = 0 + } + + t.calculated = true +} + +// Int64Slice attaches the methods of sort.Interface to []int64, sorting in increasing order. +type Int64Slice []int64 + +func (s Int64Slice) Len() int { return len(s) } +func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] } +func (s Int64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/metrics/resetting_timer_test.go b/metrics/resetting_timer_test.go new file mode 100644 index 0000000000..58fd47f352 --- /dev/null +++ b/metrics/resetting_timer_test.go @@ -0,0 +1,106 @@ +package metrics + +import ( + "testing" + "time" +) + +func TestResettingTimer(t *testing.T) { + tests := []struct { + values []int64 + start int + end int + wantP50 int64 + wantP95 int64 + wantP99 int64 + wantMean float64 + wantMin int64 + wantMax int64 + }{ + { + values: []int64{}, + start: 1, + end: 11, + wantP50: 5, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + { + values: []int64{}, + start: 1, + end: 101, + wantP50: 50, wantP95: 95, wantP99: 99, + wantMin: 1, wantMax: 100, wantMean: 50.5, + }, + { + values: []int64{1}, + start: 0, + end: 0, + wantP50: 1, wantP95: 1, wantP99: 1, + wantMin: 1, wantMax: 1, wantMean: 1, + }, + { + values: []int64{0}, + start: 0, + end: 0, + wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{}, + start: 0, + end: 0, + wantP50: 0, wantP95: 0, wantP99: 0, + wantMin: 0, wantMax: 0, wantMean: 0, + }, + { + values: []int64{1, 10}, + start: 0, + end: 0, + wantP50: 1, wantP95: 10, wantP99: 10, + wantMin: 1, wantMax: 10, wantMean: 5.5, + }, + } + for ind, tt := range tests { + timer := NewResettingTimer() + + for i := tt.start; i < tt.end; i++ { + tt.values = append(tt.values, int64(i)) + } + + for _, v := range tt.values { + timer.Update(time.Duration(v)) + } + + snap := timer.Snapshot() + + ps := snap.Percentiles([]float64{50, 95, 99}) + + val := snap.Values() + + if len(val) > 0 { + if tt.wantMin != val[0] { + t.Fatalf("%d: min: got %d, want %d", ind, val[0], tt.wantMin) + } + + if tt.wantMax != val[len(val)-1] { + t.Fatalf("%d: max: got %d, want %d", ind, val[len(val)-1], tt.wantMax) + } + } + + if tt.wantMean != snap.Mean() { + t.Fatalf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean) + } + + if tt.wantP50 != ps[0] { + t.Fatalf("%d: p50: got %d, want %d", ind, ps[0], tt.wantP50) + } + + if tt.wantP95 != ps[1] { + t.Fatalf("%d: p95: got %d, want %d", ind, ps[1], tt.wantP95) + } + + if tt.wantP99 != ps[2] { + t.Fatalf("%d: p99: got %d, want %d", ind, ps[2], tt.wantP99) + } + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/runtime.go b/metrics/runtime.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/runtime.go rename to metrics/runtime.go index 11c6b785a0..9450c479ba 100644 --- a/vendor/github.com/rcrowley/go-metrics/runtime.go +++ b/metrics/runtime.go @@ -55,7 +55,7 @@ var ( // Capture new values for the Go runtime statistics exported in // runtime.MemStats. This is designed to be called as a goroutine. func CaptureRuntimeMemStats(r Registry, d time.Duration) { - for _ = range time.Tick(d) { + for range time.Tick(d) { CaptureRuntimeMemStatsOnce(r) } } diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_cgo.go b/metrics/runtime_cgo.go similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/runtime_cgo.go rename to metrics/runtime_cgo.go diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_gccpufraction.go b/metrics/runtime_gccpufraction.go similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/runtime_gccpufraction.go rename to metrics/runtime_gccpufraction.go diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_no_cgo.go b/metrics/runtime_no_cgo.go similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/runtime_no_cgo.go rename to metrics/runtime_no_cgo.go diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_no_gccpufraction.go b/metrics/runtime_no_gccpufraction.go similarity index 100% rename from vendor/github.com/rcrowley/go-metrics/runtime_no_gccpufraction.go rename to metrics/runtime_no_gccpufraction.go diff --git a/metrics/runtime_test.go b/metrics/runtime_test.go new file mode 100644 index 0000000000..ebbfd501a2 --- /dev/null +++ b/metrics/runtime_test.go @@ -0,0 +1,88 @@ +package metrics + +import ( + "runtime" + "testing" + "time" +) + +func BenchmarkRuntimeMemStats(b *testing.B) { + r := NewRegistry() + RegisterRuntimeMemStats(r) + b.ResetTimer() + for i := 0; i < b.N; i++ { + CaptureRuntimeMemStatsOnce(r) + } +} + +func TestRuntimeMemStats(t *testing.T) { + r := NewRegistry() + RegisterRuntimeMemStats(r) + CaptureRuntimeMemStatsOnce(r) + zero := runtimeMetrics.MemStats.PauseNs.Count() // Get a "zero" since GC may have run before these tests. + runtime.GC() + CaptureRuntimeMemStatsOnce(r) + if count := runtimeMetrics.MemStats.PauseNs.Count(); 1 != count-zero { + t.Fatal(count - zero) + } + runtime.GC() + runtime.GC() + CaptureRuntimeMemStatsOnce(r) + if count := runtimeMetrics.MemStats.PauseNs.Count(); 3 != count-zero { + t.Fatal(count - zero) + } + for i := 0; i < 256; i++ { + runtime.GC() + } + CaptureRuntimeMemStatsOnce(r) + if count := runtimeMetrics.MemStats.PauseNs.Count(); 259 != count-zero { + t.Fatal(count - zero) + } + for i := 0; i < 257; i++ { + runtime.GC() + } + CaptureRuntimeMemStatsOnce(r) + if count := runtimeMetrics.MemStats.PauseNs.Count(); 515 != count-zero { // We lost one because there were too many GCs between captures. + t.Fatal(count - zero) + } +} + +func TestRuntimeMemStatsNumThread(t *testing.T) { + r := NewRegistry() + RegisterRuntimeMemStats(r) + CaptureRuntimeMemStatsOnce(r) + + if value := runtimeMetrics.NumThread.Value(); value < 1 { + t.Fatalf("got NumThread: %d, wanted at least 1", value) + } +} + +func TestRuntimeMemStatsBlocking(t *testing.T) { + if g := runtime.GOMAXPROCS(0); g < 2 { + t.Skipf("skipping TestRuntimeMemStatsBlocking with GOMAXPROCS=%d\n", g) + } + ch := make(chan int) + go testRuntimeMemStatsBlocking(ch) + var memStats runtime.MemStats + t0 := time.Now() + runtime.ReadMemStats(&memStats) + t1 := time.Now() + t.Log("i++ during runtime.ReadMemStats:", <-ch) + go testRuntimeMemStatsBlocking(ch) + d := t1.Sub(t0) + t.Log(d) + time.Sleep(d) + t.Log("i++ during time.Sleep:", <-ch) +} + +func testRuntimeMemStatsBlocking(ch chan int) { + i := 0 + for { + select { + case ch <- i: + return + default: + i++ + } + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/sample.go b/metrics/sample.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/sample.go rename to metrics/sample.go index fecee5ef68..5c4845a4f8 100644 --- a/vendor/github.com/rcrowley/go-metrics/sample.go +++ b/metrics/sample.go @@ -46,7 +46,7 @@ type ExpDecaySample struct { // NewExpDecaySample constructs a new exponentially-decaying sample with the // given reservoir size and alpha. func NewExpDecaySample(reservoirSize int, alpha float64) Sample { - if UseNilMetrics { + if !Enabled { return NilSample{} } s := &ExpDecaySample{ @@ -407,7 +407,7 @@ type UniformSample struct { // NewUniformSample constructs a new uniform sample with the given reservoir // size. func NewUniformSample(reservoirSize int) Sample { - if UseNilMetrics { + if !Enabled { return NilSample{} } return &UniformSample{ diff --git a/metrics/sample_test.go b/metrics/sample_test.go new file mode 100644 index 0000000000..d60e99c5bb --- /dev/null +++ b/metrics/sample_test.go @@ -0,0 +1,363 @@ +package metrics + +import ( + "math/rand" + "runtime" + "testing" + "time" +) + +// Benchmark{Compute,Copy}{1000,1000000} demonstrate that, even for relatively +// expensive computations like Variance, the cost of copying the Sample, as +// approximated by a make and copy, is much greater than the cost of the +// computation for small samples and only slightly less for large samples. +func BenchmarkCompute1000(b *testing.B) { + s := make([]int64, 1000) + for i := 0; i < len(s); i++ { + s[i] = int64(i) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + SampleVariance(s) + } +} +func BenchmarkCompute1000000(b *testing.B) { + s := make([]int64, 1000000) + for i := 0; i < len(s); i++ { + s[i] = int64(i) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + SampleVariance(s) + } +} +func BenchmarkCopy1000(b *testing.B) { + s := make([]int64, 1000) + for i := 0; i < len(s); i++ { + s[i] = int64(i) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + sCopy := make([]int64, len(s)) + copy(sCopy, s) + } +} +func BenchmarkCopy1000000(b *testing.B) { + s := make([]int64, 1000000) + for i := 0; i < len(s); i++ { + s[i] = int64(i) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + sCopy := make([]int64, len(s)) + copy(sCopy, s) + } +} + +func BenchmarkExpDecaySample257(b *testing.B) { + benchmarkSample(b, NewExpDecaySample(257, 0.015)) +} + +func BenchmarkExpDecaySample514(b *testing.B) { + benchmarkSample(b, NewExpDecaySample(514, 0.015)) +} + +func BenchmarkExpDecaySample1028(b *testing.B) { + benchmarkSample(b, NewExpDecaySample(1028, 0.015)) +} + +func BenchmarkUniformSample257(b *testing.B) { + benchmarkSample(b, NewUniformSample(257)) +} + +func BenchmarkUniformSample514(b *testing.B) { + benchmarkSample(b, NewUniformSample(514)) +} + +func BenchmarkUniformSample1028(b *testing.B) { + benchmarkSample(b, NewUniformSample(1028)) +} + +func TestExpDecaySample10(t *testing.T) { + rand.Seed(1) + s := NewExpDecaySample(100, 0.99) + for i := 0; i < 10; i++ { + s.Update(int64(i)) + } + if size := s.Count(); 10 != size { + t.Errorf("s.Count(): 10 != %v\n", size) + } + if size := s.Size(); 10 != size { + t.Errorf("s.Size(): 10 != %v\n", size) + } + if l := len(s.Values()); 10 != l { + t.Errorf("len(s.Values()): 10 != %v\n", l) + } + for _, v := range s.Values() { + if v > 10 || v < 0 { + t.Errorf("out of range [0, 10): %v\n", v) + } + } +} + +func TestExpDecaySample100(t *testing.T) { + rand.Seed(1) + s := NewExpDecaySample(1000, 0.01) + for i := 0; i < 100; i++ { + s.Update(int64(i)) + } + if size := s.Count(); 100 != size { + t.Errorf("s.Count(): 100 != %v\n", size) + } + if size := s.Size(); 100 != size { + t.Errorf("s.Size(): 100 != %v\n", size) + } + if l := len(s.Values()); 100 != l { + t.Errorf("len(s.Values()): 100 != %v\n", l) + } + for _, v := range s.Values() { + if v > 100 || v < 0 { + t.Errorf("out of range [0, 100): %v\n", v) + } + } +} + +func TestExpDecaySample1000(t *testing.T) { + rand.Seed(1) + s := NewExpDecaySample(100, 0.99) + for i := 0; i < 1000; i++ { + s.Update(int64(i)) + } + if size := s.Count(); 1000 != size { + t.Errorf("s.Count(): 1000 != %v\n", size) + } + if size := s.Size(); 100 != size { + t.Errorf("s.Size(): 100 != %v\n", size) + } + if l := len(s.Values()); 100 != l { + t.Errorf("len(s.Values()): 100 != %v\n", l) + } + for _, v := range s.Values() { + if v > 1000 || v < 0 { + t.Errorf("out of range [0, 1000): %v\n", v) + } + } +} + +// This test makes sure that the sample's priority is not amplified by using +// nanosecond duration since start rather than second duration since start. +// The priority becomes +Inf quickly after starting if this is done, +// effectively freezing the set of samples until a rescale step happens. +func TestExpDecaySampleNanosecondRegression(t *testing.T) { + rand.Seed(1) + s := NewExpDecaySample(100, 0.99) + for i := 0; i < 100; i++ { + s.Update(10) + } + time.Sleep(1 * time.Millisecond) + for i := 0; i < 100; i++ { + s.Update(20) + } + v := s.Values() + avg := float64(0) + for i := 0; i < len(v); i++ { + avg += float64(v[i]) + } + avg /= float64(len(v)) + if avg > 16 || avg < 14 { + t.Errorf("out of range [14, 16]: %v\n", avg) + } +} + +func TestExpDecaySampleRescale(t *testing.T) { + s := NewExpDecaySample(2, 0.001).(*ExpDecaySample) + s.update(time.Now(), 1) + s.update(time.Now().Add(time.Hour+time.Microsecond), 1) + for _, v := range s.values.Values() { + if v.k == 0.0 { + t.Fatal("v.k == 0.0") + } + } +} + +func TestExpDecaySampleSnapshot(t *testing.T) { + now := time.Now() + rand.Seed(1) + s := NewExpDecaySample(100, 0.99) + for i := 1; i <= 10000; i++ { + s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i)) + } + snapshot := s.Snapshot() + s.Update(1) + testExpDecaySampleStatistics(t, snapshot) +} + +func TestExpDecaySampleStatistics(t *testing.T) { + now := time.Now() + rand.Seed(1) + s := NewExpDecaySample(100, 0.99) + for i := 1; i <= 10000; i++ { + s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i)) + } + testExpDecaySampleStatistics(t, s) +} + +func TestUniformSample(t *testing.T) { + rand.Seed(1) + s := NewUniformSample(100) + for i := 0; i < 1000; i++ { + s.Update(int64(i)) + } + if size := s.Count(); 1000 != size { + t.Errorf("s.Count(): 1000 != %v\n", size) + } + if size := s.Size(); 100 != size { + t.Errorf("s.Size(): 100 != %v\n", size) + } + if l := len(s.Values()); 100 != l { + t.Errorf("len(s.Values()): 100 != %v\n", l) + } + for _, v := range s.Values() { + if v > 1000 || v < 0 { + t.Errorf("out of range [0, 100): %v\n", v) + } + } +} + +func TestUniformSampleIncludesTail(t *testing.T) { + rand.Seed(1) + s := NewUniformSample(100) + max := 100 + for i := 0; i < max; i++ { + s.Update(int64(i)) + } + v := s.Values() + sum := 0 + exp := (max - 1) * max / 2 + for i := 0; i < len(v); i++ { + sum += int(v[i]) + } + if exp != sum { + t.Errorf("sum: %v != %v\n", exp, sum) + } +} + +func TestUniformSampleSnapshot(t *testing.T) { + s := NewUniformSample(100) + for i := 1; i <= 10000; i++ { + s.Update(int64(i)) + } + snapshot := s.Snapshot() + s.Update(1) + testUniformSampleStatistics(t, snapshot) +} + +func TestUniformSampleStatistics(t *testing.T) { + rand.Seed(1) + s := NewUniformSample(100) + for i := 1; i <= 10000; i++ { + s.Update(int64(i)) + } + testUniformSampleStatistics(t, s) +} + +func benchmarkSample(b *testing.B, s Sample) { + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + pauseTotalNs := memStats.PauseTotalNs + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.Update(1) + } + b.StopTimer() + runtime.GC() + runtime.ReadMemStats(&memStats) + b.Logf("GC cost: %d ns/op", int(memStats.PauseTotalNs-pauseTotalNs)/b.N) +} + +func testExpDecaySampleStatistics(t *testing.T, s Sample) { + if count := s.Count(); 10000 != count { + t.Errorf("s.Count(): 10000 != %v\n", count) + } + if min := s.Min(); 107 != min { + t.Errorf("s.Min(): 107 != %v\n", min) + } + if max := s.Max(); 10000 != max { + t.Errorf("s.Max(): 10000 != %v\n", max) + } + if mean := s.Mean(); 4965.98 != mean { + t.Errorf("s.Mean(): 4965.98 != %v\n", mean) + } + if stdDev := s.StdDev(); 2959.825156930727 != stdDev { + t.Errorf("s.StdDev(): 2959.825156930727 != %v\n", stdDev) + } + ps := s.Percentiles([]float64{0.5, 0.75, 0.99}) + if 4615 != ps[0] { + t.Errorf("median: 4615 != %v\n", ps[0]) + } + if 7672 != ps[1] { + t.Errorf("75th percentile: 7672 != %v\n", ps[1]) + } + if 9998.99 != ps[2] { + t.Errorf("99th percentile: 9998.99 != %v\n", ps[2]) + } +} + +func testUniformSampleStatistics(t *testing.T, s Sample) { + if count := s.Count(); 10000 != count { + t.Errorf("s.Count(): 10000 != %v\n", count) + } + if min := s.Min(); 37 != min { + t.Errorf("s.Min(): 37 != %v\n", min) + } + if max := s.Max(); 9989 != max { + t.Errorf("s.Max(): 9989 != %v\n", max) + } + if mean := s.Mean(); 4748.14 != mean { + t.Errorf("s.Mean(): 4748.14 != %v\n", mean) + } + if stdDev := s.StdDev(); 2826.684117548333 != stdDev { + t.Errorf("s.StdDev(): 2826.684117548333 != %v\n", stdDev) + } + ps := s.Percentiles([]float64{0.5, 0.75, 0.99}) + if 4599 != ps[0] { + t.Errorf("median: 4599 != %v\n", ps[0]) + } + if 7380.5 != ps[1] { + t.Errorf("75th percentile: 7380.5 != %v\n", ps[1]) + } + if 9986.429999999998 != ps[2] { + t.Errorf("99th percentile: 9986.429999999998 != %v\n", ps[2]) + } +} + +// TestUniformSampleConcurrentUpdateCount would expose data race problems with +// concurrent Update and Count calls on Sample when test is called with -race +// argument +func TestUniformSampleConcurrentUpdateCount(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + s := NewUniformSample(100) + for i := 0; i < 100; i++ { + s.Update(int64(i)) + } + quit := make(chan struct{}) + go func() { + t := time.NewTicker(10 * time.Millisecond) + for { + select { + case <-t.C: + s.Update(rand.Int63()) + case <-quit: + t.Stop() + return + } + } + }() + for i := 0; i < 1000; i++ { + s.Count() + time.Sleep(5 * time.Millisecond) + } + quit <- struct{}{} +} diff --git a/vendor/github.com/rcrowley/go-metrics/syslog.go b/metrics/syslog.go similarity index 98% rename from vendor/github.com/rcrowley/go-metrics/syslog.go rename to metrics/syslog.go index 693f190855..a0ed4b1b23 100644 --- a/vendor/github.com/rcrowley/go-metrics/syslog.go +++ b/metrics/syslog.go @@ -11,7 +11,7 @@ import ( // Output each metric in the given registry to syslog periodically using // the given syslogger. func Syslog(r Registry, d time.Duration, w *syslog.Writer) { - for _ = range time.Tick(d) { + for range time.Tick(d) { r.Each(func(name string, i interface{}) { switch metric := i.(type) { case Counter: diff --git a/vendor/github.com/rcrowley/go-metrics/timer.go b/metrics/timer.go similarity index 93% rename from vendor/github.com/rcrowley/go-metrics/timer.go rename to metrics/timer.go index 17db8f8d20..89e22208fd 100644 --- a/vendor/github.com/rcrowley/go-metrics/timer.go +++ b/metrics/timer.go @@ -19,6 +19,7 @@ type Timer interface { RateMean() float64 Snapshot() Timer StdDev() float64 + Stop() Sum() int64 Time(func()) Update(time.Duration) @@ -28,6 +29,8 @@ type Timer interface { // GetOrRegisterTimer returns an existing Timer or constructs and registers a // new StandardTimer. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. func GetOrRegisterTimer(name string, r Registry) Timer { if nil == r { r = DefaultRegistry @@ -36,8 +39,9 @@ func GetOrRegisterTimer(name string, r Registry) Timer { } // NewCustomTimer constructs a new StandardTimer from a Histogram and a Meter. +// Be sure to call Stop() once the timer is of no use to allow for garbage collection. func NewCustomTimer(h Histogram, m Meter) Timer { - if UseNilMetrics { + if !Enabled { return NilTimer{} } return &StandardTimer{ @@ -47,6 +51,8 @@ func NewCustomTimer(h Histogram, m Meter) Timer { } // NewRegisteredTimer constructs and registers a new StandardTimer. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. func NewRegisteredTimer(name string, r Registry) Timer { c := NewTimer() if nil == r { @@ -58,8 +64,9 @@ func NewRegisteredTimer(name string, r Registry) Timer { // NewTimer constructs a new StandardTimer using an exponentially-decaying // sample with the same reservoir size and alpha as UNIX load averages. +// Be sure to call Stop() once the timer is of no use to allow for garbage collection. func NewTimer() Timer { - if UseNilMetrics { + if !Enabled { return NilTimer{} } return &StandardTimer{ @@ -112,6 +119,9 @@ func (NilTimer) Snapshot() Timer { return NilTimer{} } // StdDev is a no-op. func (NilTimer) StdDev() float64 { return 0.0 } +// Stop is a no-op. +func (NilTimer) Stop() {} + // Sum is a no-op. func (NilTimer) Sum() int64 { return 0 } @@ -201,6 +211,11 @@ func (t *StandardTimer) StdDev() float64 { return t.histogram.StdDev() } +// Stop stops the meter. +func (t *StandardTimer) Stop() { + t.meter.Stop() +} + // Sum returns the sum in the sample. func (t *StandardTimer) Sum() int64 { return t.histogram.Sum() @@ -288,6 +303,9 @@ func (t *TimerSnapshot) Snapshot() Timer { return t } // was taken. func (t *TimerSnapshot) StdDev() float64 { return t.histogram.StdDev() } +// Stop is a no-op. +func (t *TimerSnapshot) Stop() {} + // Sum returns the sum at the time the snapshot was taken. func (t *TimerSnapshot) Sum() int64 { return t.histogram.Sum() } diff --git a/metrics/timer_test.go b/metrics/timer_test.go new file mode 100644 index 0000000000..f85c9b803d --- /dev/null +++ b/metrics/timer_test.go @@ -0,0 +1,101 @@ +package metrics + +import ( + "fmt" + "math" + "testing" + "time" +) + +func BenchmarkTimer(b *testing.B) { + tm := NewTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + tm.Update(1) + } +} + +func TestGetOrRegisterTimer(t *testing.T) { + r := NewRegistry() + NewRegisteredTimer("foo", r).Update(47) + if tm := GetOrRegisterTimer("foo", r); 1 != tm.Count() { + t.Fatal(tm) + } +} + +func TestTimerExtremes(t *testing.T) { + tm := NewTimer() + tm.Update(math.MaxInt64) + tm.Update(0) + if stdDev := tm.StdDev(); 4.611686018427388e+18 != stdDev { + t.Errorf("tm.StdDev(): 4.611686018427388e+18 != %v\n", stdDev) + } +} + +func TestTimerStop(t *testing.T) { + l := len(arbiter.meters) + tm := NewTimer() + if len(arbiter.meters) != l+1 { + t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters)) + } + tm.Stop() + if len(arbiter.meters) != l { + t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters)) + } +} + +func TestTimerFunc(t *testing.T) { + tm := NewTimer() + tm.Time(func() { time.Sleep(50e6) }) + if max := tm.Max(); 45e6 > max || max > 55e6 { + t.Errorf("tm.Max(): 45e6 > %v || %v > 55e6\n", max, max) + } +} + +func TestTimerZero(t *testing.T) { + tm := NewTimer() + if count := tm.Count(); 0 != count { + t.Errorf("tm.Count(): 0 != %v\n", count) + } + if min := tm.Min(); 0 != min { + t.Errorf("tm.Min(): 0 != %v\n", min) + } + if max := tm.Max(); 0 != max { + t.Errorf("tm.Max(): 0 != %v\n", max) + } + if mean := tm.Mean(); 0.0 != mean { + t.Errorf("tm.Mean(): 0.0 != %v\n", mean) + } + if stdDev := tm.StdDev(); 0.0 != stdDev { + t.Errorf("tm.StdDev(): 0.0 != %v\n", stdDev) + } + ps := tm.Percentiles([]float64{0.5, 0.75, 0.99}) + if 0.0 != ps[0] { + t.Errorf("median: 0.0 != %v\n", ps[0]) + } + if 0.0 != ps[1] { + t.Errorf("75th percentile: 0.0 != %v\n", ps[1]) + } + if 0.0 != ps[2] { + t.Errorf("99th percentile: 0.0 != %v\n", ps[2]) + } + if rate1 := tm.Rate1(); 0.0 != rate1 { + t.Errorf("tm.Rate1(): 0.0 != %v\n", rate1) + } + if rate5 := tm.Rate5(); 0.0 != rate5 { + t.Errorf("tm.Rate5(): 0.0 != %v\n", rate5) + } + if rate15 := tm.Rate15(); 0.0 != rate15 { + t.Errorf("tm.Rate15(): 0.0 != %v\n", rate15) + } + if rateMean := tm.RateMean(); 0.0 != rateMean { + t.Errorf("tm.RateMean(): 0.0 != %v\n", rateMean) + } +} + +func ExampleGetOrRegisterTimer() { + m := "account.create.latency" + t := GetOrRegisterTimer(m, nil) + t.Update(47) + fmt.Println(t.Max()) // Output: 47 +} diff --git a/vendor/github.com/rcrowley/go-metrics/validate.sh b/metrics/validate.sh similarity index 93% rename from vendor/github.com/rcrowley/go-metrics/validate.sh rename to metrics/validate.sh index f6499982e5..c4ae91e642 100755 --- a/vendor/github.com/rcrowley/go-metrics/validate.sh +++ b/metrics/validate.sh @@ -7,4 +7,4 @@ GOFMT_LINES=`gofmt -l . | wc -l | xargs` test $GOFMT_LINES -eq 0 || echo "gofmt needs to be run, ${GOFMT_LINES} files have issues" # run the tests for the root package -go test . +go test -race . diff --git a/vendor/github.com/rcrowley/go-metrics/writer.go b/metrics/writer.go similarity index 99% rename from vendor/github.com/rcrowley/go-metrics/writer.go rename to metrics/writer.go index 091e971d2e..88521a80d9 100644 --- a/vendor/github.com/rcrowley/go-metrics/writer.go +++ b/metrics/writer.go @@ -10,7 +10,7 @@ import ( // Write sorts writes each metric in the given registry periodically to the // given io.Writer. func Write(r Registry, d time.Duration, w io.Writer) { - for _ = range time.Tick(d) { + for range time.Tick(d) { WriteOnce(r, w) } } diff --git a/metrics/writer_test.go b/metrics/writer_test.go new file mode 100644 index 0000000000..1aacc28712 --- /dev/null +++ b/metrics/writer_test.go @@ -0,0 +1,22 @@ +package metrics + +import ( + "sort" + "testing" +) + +func TestMetricsSorting(t *testing.T) { + var namedMetrics = namedMetricSlice{ + {name: "zzz"}, + {name: "bbb"}, + {name: "fff"}, + {name: "ggg"}, + } + + sort.Sort(namedMetrics) + for i, name := range []string{"bbb", "fff", "ggg", "zzz"} { + if namedMetrics[i].name != name { + t.Fail() + } + } +} diff --git a/node/api.go b/node/api.go index 4e9b1edc47..992a7c4166 100644 --- a/node/api.go +++ b/node/api.go @@ -24,10 +24,10 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/rpc" - "github.com/rcrowley/go-metrics" ) // PrivateAdminAPI is the collection of administrative API methods exposed only diff --git a/p2p/message.go b/p2p/message.go index 5690494bf4..50b419970d 100644 --- a/p2p/message.go +++ b/p2p/message.go @@ -22,8 +22,6 @@ import ( "fmt" "io" "io/ioutil" - "net" - "sync" "sync/atomic" "time" @@ -112,30 +110,6 @@ func SendItems(w MsgWriter, msgcode uint64, elems ...interface{}) error { return Send(w, msgcode, elems) } -// netWrapper wraps a MsgReadWriter with locks around -// ReadMsg/WriteMsg and applies read/write deadlines. -type netWrapper struct { - rmu, wmu sync.Mutex - - rtimeout, wtimeout time.Duration - conn net.Conn - wrapped MsgReadWriter -} - -func (rw *netWrapper) ReadMsg() (Msg, error) { - rw.rmu.Lock() - defer rw.rmu.Unlock() - rw.conn.SetReadDeadline(time.Now().Add(rw.rtimeout)) - return rw.wrapped.ReadMsg() -} - -func (rw *netWrapper) WriteMsg(msg Msg) error { - rw.wmu.Lock() - defer rw.wmu.Unlock() - rw.conn.SetWriteDeadline(time.Now().Add(rw.wtimeout)) - return rw.wrapped.WriteMsg(msg) -} - // eofSignal wraps a reader with eof signaling. the eof channel is // closed when the wrapped reader returns an error or when count bytes // have been read. diff --git a/p2p/metrics.go b/p2p/metrics.go index 98b61901d5..4cbff90aca 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -25,10 +25,10 @@ import ( ) var ( - ingressConnectMeter = metrics.NewMeter("p2p/InboundConnects") - ingressTrafficMeter = metrics.NewMeter("p2p/InboundTraffic") - egressConnectMeter = metrics.NewMeter("p2p/OutboundConnects") - egressTrafficMeter = metrics.NewMeter("p2p/OutboundTraffic") + ingressConnectMeter = metrics.NewRegisteredMeter("p2p/InboundConnects", nil) + ingressTrafficMeter = metrics.NewRegisteredMeter("p2p/InboundTraffic", nil) + egressConnectMeter = metrics.NewRegisteredMeter("p2p/OutboundConnects", nil) + egressTrafficMeter = metrics.NewRegisteredMeter("p2p/OutboundTraffic", nil) ) // meteredConn is a wrapper around a network TCP connection that meters both the diff --git a/p2p/protocols/protocol.go b/p2p/protocols/protocol.go index bb934ca45a..9914c99587 100644 --- a/p2p/protocols/protocol.go +++ b/p2p/protocols/protocol.go @@ -22,8 +22,8 @@ devp2p subprotocols by abstracting away code standardly shared by protocols. * automate RLP decoding/encoding based on reflecting * provide the forever loop to read incoming messages * standardise error handling related to communication +* standardised handshake negotiation * TODO: automatic generation of wire protocol specification for peers -* standardise handshake negotiation */ package protocols @@ -84,7 +84,7 @@ type Error struct { } func (e Error) Error() (message string) { - if len(message) == 0 { + if len(e.message) == 0 { name, ok := errorToString[e.Code] if !ok { panic("invalid message code") @@ -98,13 +98,11 @@ func (e Error) Error() (message string) { } func errorf(code int, format string, params ...interface{}) *Error { - e := &Error{ + return &Error{ Code: code, format: format, params: params, } - - return e } // Spec is a protocol specification including its name and version as well as @@ -119,10 +117,11 @@ type Spec struct { // MaxMsgSize is the maximum accepted length of the message payload MaxMsgSize uint32 - // Messages is a list of message types which this protocol uses, with + // Messages is a list of message data types which this protocol uses, with // each message type being sent with its array index as the code (so // [&foo{}, &bar{}, &baz{}] would send foo, bar and baz with codes // 0, 1 and 2 respectively) + // each message must have a single unique data type Messages []interface{} initOnce sync.Once @@ -183,8 +182,8 @@ type Peer struct { // NewPeer constructs a new peer // this constructor is called by the p2p.Protocol#Run function -// the first two arguments are coming the arguments passed to p2p.Protocol.Run function -// the third argument is the CodeMap describing the protocol messages and options +// the first two arguments are the arguments passed to p2p.Protocol.Run function +// the third argument is the Spec describing the protocol func NewPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *Spec) *Peer { return &Peer{ Peer: p, @@ -195,6 +194,9 @@ func NewPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *Spec) *Peer { // Run starts the forever loop that handles incoming messages // called within the p2p.Protocol#Run function +// the handler argument is a function which is called for each message received +// from the remote peer, a returned error causes the loop to exit +// resulting in disconnection func (p *Peer) Run(handler func(msg interface{}) error) error { for { if err := p.handleIncoming(handler); err != nil { @@ -204,11 +206,8 @@ func (p *Peer) Run(handler func(msg interface{}) error) error { } // Drop disconnects a peer. -// falls back to self.disconnect which is set as p2p.Peer#Disconnect except -// for test peers where it calls p2p.MsgPipe#Close so that the readloop can terminate // TODO: may need to implement protocol drop only? don't want to kick off the peer // if they are useful for other protocols -// overwrite Disconnect for testing, so that protocol readloop quits func (p *Peer) Drop(err error) { p.Disconnect(p2p.DiscSubprotocolError) } @@ -222,7 +221,6 @@ func (p *Peer) Send(msg interface{}) error { if !found { return errorf(ErrInvalidMsgType, "%v", code) } - // log.Trace(fmt.Sprintf("=> msg %s#%d TO %v : %v", p.spec.Name, code, p.ID(), msg)) return p2p.Send(p.rw, code, msg) } @@ -253,7 +251,6 @@ func (p *Peer) handleIncoming(handle func(msg interface{}) error) error { if err := msg.Decode(val); err != nil { return errorf(ErrDecode, "<= %v: %v", msg, err) } - // log.Trace(fmt.Sprintf("<= %s/%v FROM %v %T %v", p.spec.Name, msg, p.ID(), val, val)) // call the registered handler callbacks // a registered callback take the decoded message as argument as an interface @@ -274,7 +271,7 @@ func (p *Peer) handleIncoming(handle func(msg interface{}) error) error { // * expects a remote handshake back of the same type // * the dialing peer needs to send the handshake first and then waits for remote // * the listening peer waits for the remote handshake and then sends it -// returns the remote hs and an error +// returns the remote handshake and an error func (p *Peer) Handshake(ctx context.Context, hs interface{}, verify func(interface{}) error) (rhs interface{}, err error) { if _, ok := p.spec.GetCode(hs); !ok { return nil, errorf(ErrHandshake, "unknown handshake message type: %T", hs) @@ -289,13 +286,18 @@ func (p *Peer) Handshake(ctx context.Context, hs interface{}, verify func(interf } send := func() { errc <- p.Send(hs) } receive := func() { errc <- p.handleIncoming(handle) } - var last bool - for { - if p.Inbound() == last { - go send() + + go func() { + if p.Inbound() { + receive() + send() } else { - go receive() + send() + receive() } + }() + + for i := 0; i < 2; i++ { select { case err = <-errc: case <-ctx.Done(): @@ -304,10 +306,6 @@ func (p *Peer) Handshake(ctx context.Context, hs interface{}, verify func(interf if err != nil { return nil, errorf(ErrHandshake, err.Error()) } - if last { - break - } - last = true } return rhs, nil } diff --git a/p2p/protocols/protocol_test.go b/p2p/protocols/protocol_test.go index 1a5c480386..aaae7502b5 100644 --- a/p2p/protocols/protocol_test.go +++ b/p2p/protocols/protocol_test.go @@ -20,21 +20,15 @@ import ( "context" "errors" "fmt" - "os" "testing" "time" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" p2ptest "github.com/ethereum/go-ethereum/p2p/testing" ) -func init() { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlWarn, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) -} - // handshake message type type hs0 struct { C uint @@ -141,8 +135,7 @@ func newProtocol(pp *p2ptest.TestPeerPool) func(*p2p.Peer, p2p.MsgReadWriter) er pp.Add(peer) defer pp.Remove(peer) - err = peer.Run(handle) - return err + return peer.Run(handle) } } @@ -232,8 +225,12 @@ func runModuleHandshake(t *testing.T, resp uint, errs ...error) { pp := p2ptest.NewTestPeerPool() s := protocolTester(t, pp) id := s.IDs[0] - s.TestExchanges(protoHandshakeExchange(id, &protoHandshake{42, "420"})...) - s.TestExchanges(moduleHandshakeExchange(id, resp)...) + if err := s.TestExchanges(protoHandshakeExchange(id, &protoHandshake{42, "420"})...); err != nil { + t.Fatal(err) + } + if err := s.TestExchanges(moduleHandshakeExchange(id, resp)...); err != nil { + t.Fatal(err) + } var disconnects []*p2ptest.Disconnect for i, err := range errs { disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err}) @@ -308,29 +305,32 @@ func runMultiplePeers(t *testing.T, peer int, errs ...error) { pp := p2ptest.NewTestPeerPool() s := protocolTester(t, pp) - s.TestExchanges(testMultiPeerSetup(s.IDs[0], s.IDs[1])...) + if err := s.TestExchanges(testMultiPeerSetup(s.IDs[0], s.IDs[1])...); err != nil { + t.Fatal(err) + } // after some exchanges of messages, we can test state changes // here this is simply demonstrated by the peerPool // after the handshake negotiations peers must be added to the pool // time.Sleep(1) - for !pp.Has(s.IDs[0]) { - time.Sleep(1) - log.Trace(fmt.Sprintf("missing peer test-0: %v (%v)", pp, s.IDs)) + tick := time.NewTicker(10 * time.Millisecond) + timeout := time.NewTimer(1 * time.Second) +WAIT: + for { + select { + case <-tick.C: + if pp.Has(s.IDs[0]) { + break WAIT + } + case <-timeout.C: + t.Fatal("timeout") + } } - if !pp.Has(s.IDs[0]) { - t.Fatalf("missing peer test-0: %v (%v)", pp, s.IDs) - } - for !pp.Has(s.IDs[1]) { - time.Sleep(1) - log.Trace(fmt.Sprintf("missing peer test-1: %v (%v)", pp, s.IDs)) - } - if !pp.Has(s.IDs[1]) { t.Fatalf("missing peer test-1: %v (%v)", pp, s.IDs) } // peer 0 sends kill request for peer with index
- s.TestExchanges(p2ptest.Exchange{ + err := s.TestExchanges(p2ptest.Exchange{ Triggers: []p2ptest.Trigger{ { Code: 2, @@ -340,8 +340,12 @@ func runMultiplePeers(t *testing.T, peer int, errs ...error) { }, }) + if err != nil { + t.Fatal(err) + } + // the peer not killed sends a drop request - s.TestExchanges(p2ptest.Exchange{ + err = s.TestExchanges(p2ptest.Exchange{ Triggers: []p2ptest.Trigger{ { Code: 3, @@ -350,6 +354,11 @@ func runMultiplePeers(t *testing.T, peer int, errs ...error) { }, }, }) + + if err != nil { + t.Fatal(err) + } + // check the actual discconnect errors on the individual peers var disconnects []*p2ptest.Disconnect for i, err := range errs { @@ -364,7 +373,6 @@ func runMultiplePeers(t *testing.T, peer int, errs ...error) { } } - func XTestMultiplePeersDropSelf(t *testing.T) { runMultiplePeers(t, 0, fmt.Errorf("subprotocol error"), diff --git a/p2p/server.go b/p2p/server.go index 90e92dc05b..c41d1dc156 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -36,9 +36,7 @@ import ( ) const ( - defaultDialTimeout = 15 * time.Second - refreshPeersInterval = 30 * time.Second - staticPeerCheckInterval = 15 * time.Second + defaultDialTimeout = 15 * time.Second // Connectivity defaults. maxActiveDialTasks = 16 diff --git a/p2p/testing/protocolsession.go b/p2p/testing/protocolsession.go index 8d19f4b3f2..6547bf250c 100644 --- a/p2p/testing/protocolsession.go +++ b/p2p/testing/protocolsession.go @@ -28,6 +28,11 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations/adapters" ) +var errTimedOut = errors.New("timed out") + +// ProtocolSession is a quasi simulation of a pivot node running +// a service and a number of dummy peers that can send (trigger) or +// receive (expect) messages type ProtocolSession struct { Server *p2p.Server IDs []discover.NodeID @@ -35,9 +40,9 @@ type ProtocolSession struct { events chan *p2p.PeerEvent } -// exchanges are the basic units of protocol tests +// Exchange is the basic units of protocol tests // the triggers and expects in the arrays are run immediately and asynchronously -// thus one cannot have multiple expects for the SAME peer with the DIFFERENT messagetypes +// thus one cannot have multiple expects for the SAME peer with DIFFERENT message types // because it's unpredictable which expect will receive which message // (with expect #1 and #2, messages might be sent #2 and #1, and both expects will complain about wrong message code) // an exchange is defined on a session @@ -45,9 +50,11 @@ type Exchange struct { Label string Triggers []Trigger Expects []Expect + Timeout time.Duration } -// part of the exchange, incoming message from a set of peers +// Trigger is part of the exchange, incoming message for the pivot node +// sent by a peer type Trigger struct { Msg interface{} // type of message to be sent Code uint64 // code of message is given @@ -55,6 +62,8 @@ type Trigger struct { Timeout time.Duration // timeout duration for the sending } +// Expect is part of an exchange, outgoing message from the pivot node +// received by a peer type Expect struct { Msg interface{} // type of message to expect Code uint64 // code of message is now given @@ -62,6 +71,7 @@ type Expect struct { Timeout time.Duration // timeout duration for receiving } +// Disconnect represents a disconnect event, used and checked by TestDisconnected type Disconnect struct { Peer discover.NodeID // discconnected peer Error error // disconnect reason @@ -90,108 +100,158 @@ func (self *ProtocolSession) trigger(trig Trigger) error { if t == time.Duration(0) { t = 1000 * time.Millisecond } - alarm := time.NewTimer(t) select { case err := <-errc: return err - case <-alarm.C: + case <-time.After(t): return fmt.Errorf("timout expecting %v to send to peer %v", trig.Msg, trig.Peer) } } -// expect checks an expectation -func (self *ProtocolSession) expect(exp Expect) error { - if exp.Msg == nil { - return errors.New("no message to expect") - } - simNode, ok := self.adapter.GetNode(exp.Peer) - if !ok { - return fmt.Errorf("trigger: peer %v does not exist (1- %v)", exp.Peer, len(self.IDs)) - } - mockNode, ok := simNode.Services()[0].(*mockNode) - if !ok { - return fmt.Errorf("trigger: peer %v is not a mock", exp.Peer) +// expect checks an expectation of a message sent out by the pivot node +func (self *ProtocolSession) expect(exps []Expect) error { + // construct a map of expectations for each node + peerExpects := make(map[discover.NodeID][]Expect) + for _, exp := range exps { + if exp.Msg == nil { + return errors.New("no message to expect") + } + peerExpects[exp.Peer] = append(peerExpects[exp.Peer], exp) } + // construct a map of mockNodes for each node + mockNodes := make(map[discover.NodeID]*mockNode) + for nodeID := range peerExpects { + simNode, ok := self.adapter.GetNode(nodeID) + if !ok { + return fmt.Errorf("trigger: peer %v does not exist (1- %v)", nodeID, len(self.IDs)) + } + mockNode, ok := simNode.Services()[0].(*mockNode) + if !ok { + return fmt.Errorf("trigger: peer %v is not a mock", nodeID) + } + mockNodes[nodeID] = mockNode + } + + // done chanell cancels all created goroutines when function returns + done := make(chan struct{}) + defer close(done) + // errc catches the first error from errc := make(chan error) + + wg := &sync.WaitGroup{} + wg.Add(len(mockNodes)) + for nodeID, mockNode := range mockNodes { + nodeID := nodeID + mockNode := mockNode + go func() { + defer wg.Done() + + // Sum all Expect timeouts to give the maximum + // time for all expectations to finish. + // mockNode.Expect checks all received messages against + // a list of expected messages and timeout for each + // of them can not be checked separately. + var t time.Duration + for _, exp := range peerExpects[nodeID] { + if exp.Timeout == time.Duration(0) { + t += 2000 * time.Millisecond + } else { + t += exp.Timeout + } + } + alarm := time.NewTimer(t) + defer alarm.Stop() + + // expectErrc is used to check if error returned + // from mockNode.Expect is not nil and to send it to + // errc only in that case. + // done channel will be closed when function + expectErrc := make(chan error) + go func() { + select { + case expectErrc <- mockNode.Expect(peerExpects[nodeID]...): + case <-done: + case <-alarm.C: + } + }() + + select { + case err := <-expectErrc: + if err != nil { + select { + case errc <- err: + case <-done: + case <-alarm.C: + errc <- errTimedOut + } + } + case <-done: + case <-alarm.C: + errc <- errTimedOut + } + + }() + } + go func() { - log.Trace(fmt.Sprintf("waiting for msg, %v", exp.Msg)) - errc <- mockNode.Expect(&exp) + wg.Wait() + // close errc when all goroutines finish to return nill err from errc + close(errc) }() - t := exp.Timeout - if t == time.Duration(0) { + return <-errc +} + +// TestExchanges tests a series of exchanges against the session +func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { + for i, e := range exchanges { + if err := self.testExchange(e); err != nil { + return fmt.Errorf("exchange #%d %q: %v", i, e.Label, err) + } + log.Trace(fmt.Sprintf("exchange #%d %q: run successfully", i, e.Label)) + } + return nil +} + +// testExchange tests a single Exchange. +// Default timeout value is 2 seconds. +func (self *ProtocolSession) testExchange(e Exchange) error { + errc := make(chan error) + done := make(chan struct{}) + defer close(done) + + go func() { + for _, trig := range e.Triggers { + err := self.trigger(trig) + if err != nil { + errc <- err + return + } + } + + select { + case errc <- self.expect(e.Expects): + case <-done: + } + }() + + // time out globally or finish when all expectations satisfied + t := e.Timeout + if t == 0 { t = 2000 * time.Millisecond } alarm := time.NewTimer(t) select { case err := <-errc: - log.Trace(fmt.Sprintf("expected msg arrives with error %v", err)) return err case <-alarm.C: - return fmt.Errorf("timout expecting %v sent to peer %v", exp.Msg, exp.Peer) + return errTimedOut } } -// TestExchanges tests a series of exchanges againsts the session -func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error { - // launch all triggers of this exchanges - - for i, e := range exchanges { - errc := make(chan error) - wg := &sync.WaitGroup{} - for _, trig := range e.Triggers { - err := self.trigger(trig) - if err != nil { - errc <- err - } - } - - // each expectation is spawned in separate go-routine - // expectations of an exchange are conjunctive but unordered, i.e., - // only all of them arriving constitutes a pass - // each expectation is meant to be for a different peer, otherwise they are expected to panic - // testing of an exchange blocks until all expectations are decided - // an expectation is decided if - // expected message arrives OR - // an unexpected message arrives (panic) - // times out on their individual timeout - for _, ex := range e.Expects { - wg.Add(1) - // expect msg spawned to separate go routine - go func(exp Expect) { - defer wg.Done() - err := self.expect(exp) - if err != nil { - log.Trace(fmt.Sprintf("expect msg fails %v", err)) - errc <- err - } - }(ex) - } - - // wait for all expectations - go func() { - wg.Wait() - close(errc) - }() - - // time out globally or finish when all expectations satisfied - alarm := time.NewTimer(1000 * time.Millisecond) - select { - - case err := <-errc: - if err != nil { - return fmt.Errorf("exchange failed with: %v", err) - } else { - log.Trace(fmt.Sprintf("exchange %v: '%v' run successfully", i, e.Label)) - } - case <-alarm.C: - return fmt.Errorf("exchange %v: '%v' timed out", i, e.Label) - } - } - return nil -} - +// TestDisconnected tests the disconnections given as arguments +// the disconnect structs describe what disconnect error is expected on which peer func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error { expects := make(map[discover.NodeID]error) for _, disconnect := range disconnects { @@ -209,10 +269,8 @@ func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error if !ok { continue } - log.Trace("disconnects: ", "peer", event.Peer, "event type", event.Type, "expect", expectErr, "error", event.Error) if !(expectErr == nil && event.Error == "" || expectErr != nil && expectErr.Error() == event.Error) { - log.Trace("error!!!") return fmt.Errorf("unexpected error on peer %v. expected '%v', got '%v'", event.Peer, expectErr, event.Error) } delete(expects, event.Peer) diff --git a/p2p/testing/protocoltester.go b/p2p/testing/protocoltester.go index 777360f1d6..a797412d60 100644 --- a/p2p/testing/protocoltester.go +++ b/p2p/testing/protocoltester.go @@ -15,13 +15,20 @@ // along with the go-ethereum library. If not, see
. /* -TODO: documentation +the p2p/testing package provides a unit test scheme to check simple +protocol message exchanges with one pivot node and a number of dummy peers +The pivot test node runs a node.Service, the dummy peers run a mock node +that can be used to send and receive messages */ package testing import ( + "bytes" "fmt" + "io" + "io/ioutil" + "strings" "sync" "testing" @@ -31,14 +38,20 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" ) +// ProtocolTester is the tester environment used for unit testing protocol +// message exchanges. It uses p2p/simulations framework type ProtocolTester struct { *ProtocolSession network *simulations.Network } +// NewProtocolTester constructs a new ProtocolTester +// it takes as argument the pivot node id, the number of dummy peers and the +// protocol run function called on a peer connection by the p2p server func NewProtocolTester(t *testing.T, id discover.NodeID, n int, run func(*p2p.Peer, p2p.MsgReadWriter) error) *ProtocolTester { services := adapters.Services{ "test": func(ctx *adapters.ServiceContext) (node.Service, error) { @@ -87,11 +100,14 @@ func NewProtocolTester(t *testing.T, id discover.NodeID, n int, run func(*p2p.Pe return self } +// Stop stops the p2p server func (self *ProtocolTester) Stop() error { self.Server.Stop() return nil } +// Connect brings up the remote peer node and connects it using the +// p2p/simulations network connection with the in memory network adapter func (self *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) { for _, peer := range peers { log.Trace(fmt.Sprintf("start node %v", peer.ID)) @@ -141,7 +157,7 @@ type mockNode struct { testNode trigger chan *Trigger - expect chan *Expect + expect chan []Expect err chan error stop chan struct{} stopOnce sync.Once @@ -150,7 +166,7 @@ type mockNode struct { func newMockNode() *mockNode { mock := &mockNode{ trigger: make(chan *Trigger), - expect: make(chan *Expect), + expect: make(chan []Expect), err: make(chan error), stop: make(chan struct{}), } @@ -165,8 +181,8 @@ func (m *mockNode) Run(peer *p2p.Peer, rw p2p.MsgReadWriter) error { select { case trig := <-m.trigger: m.err <- p2p.Send(rw, trig.Code, trig.Msg) - case exp := <-m.expect: - m.err <- p2p.ExpectMsg(rw, exp.Code, exp.Msg) + case exps := <-m.expect: + m.err <- expectMsgs(rw, exps) case <-m.stop: return nil } @@ -178,7 +194,7 @@ func (m *mockNode) Trigger(trig *Trigger) error { return <-m.err } -func (m *mockNode) Expect(exp *Expect) error { +func (m *mockNode) Expect(exp ...Expect) error { m.expect <- exp return <-m.err } @@ -187,3 +203,67 @@ func (m *mockNode) Stop() error { m.stopOnce.Do(func() { close(m.stop) }) return nil } + +func expectMsgs(rw p2p.MsgReadWriter, exps []Expect) error { + matched := make([]bool, len(exps)) + for { + msg, err := rw.ReadMsg() + if err != nil { + if err == io.EOF { + break + } + return err + } + actualContent, err := ioutil.ReadAll(msg.Payload) + if err != nil { + return err + } + var found bool + for i, exp := range exps { + if exp.Code == msg.Code && bytes.Equal(actualContent, mustEncodeMsg(exp.Msg)) { + if matched[i] { + return fmt.Errorf("message #%d received two times", i) + } + matched[i] = true + found = true + break + } + } + if !found { + expected := make([]string, 0) + for i, exp := range exps { + if matched[i] { + continue + } + expected = append(expected, fmt.Sprintf("code %d payload %x", exp.Code, mustEncodeMsg(exp.Msg))) + } + return fmt.Errorf("unexpected message code %d payload %x, expected %s", msg.Code, actualContent, strings.Join(expected, " or ")) + } + done := true + for _, m := range matched { + if !m { + done = false + break + } + } + if done { + return nil + } + } + for i, m := range matched { + if !m { + return fmt.Errorf("expected message #%d not received", i) + } + } + return nil +} + +// mustEncodeMsg uses rlp to encode a message. +// In case of error it panics. +func mustEncodeMsg(msg interface{}) []byte { + contentEnc, err := rlp.EncodeToBytes(msg) + if err != nil { + panic("content encode error: " + err.Error()) + } + return contentEnc +} diff --git a/params/config.go b/params/config.go index 345f6394ab..dc02c7ca37 100644 --- a/params/config.go +++ b/params/config.go @@ -31,46 +31,46 @@ var ( var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(1150000), - DAOForkBlock: big.NewInt(1920000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2463000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2675000), - EIP158Block: big.NewInt(2675000), - ByzantiumBlock: big.NewInt(4370000), - - Ethash: new(EthashConfig), + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(1150000), + DAOForkBlock: big.NewInt(1920000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2463000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2675000), + EIP158Block: big.NewInt(2675000), + ByzantiumBlock: big.NewInt(4370000), + ConstantinopleBlock: nil, + Ethash: new(EthashConfig), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. TestnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - ByzantiumBlock: big.NewInt(1700000), - - Ethash: new(EthashConfig), + ChainId: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + ByzantiumBlock: big.NewInt(1700000), + ConstantinopleBlock: nil, + Ethash: new(EthashConfig), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. RinkebyChainConfig = &ChainConfig{ - ChainId: big.NewInt(4), - HomesteadBlock: big.NewInt(1), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(2), - EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), - EIP155Block: big.NewInt(3), - EIP158Block: big.NewInt(3), - ByzantiumBlock: big.NewInt(1035301), - + ChainId: big.NewInt(4), + HomesteadBlock: big.NewInt(1), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(2), + EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), + EIP155Block: big.NewInt(3), + EIP158Block: big.NewInt(3), + ByzantiumBlock: big.NewInt(1035301), + ConstantinopleBlock: nil, Clique: &CliqueConfig{ Period: 15, Epoch: 30000, @@ -82,16 +82,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -115,7 +115,8 @@ type ChainConfig struct { EIP155Block *big.Int `json:"eip155Block,omitempty"` // EIP155 HF block EIP158Block *big.Int `json:"eip158Block,omitempty"` // EIP158 HF block - ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = already on byzantium) + ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = already on byzantium) + ConstantinopleBlock *big.Int `json:"constantinopleBlock,omitempty"` // Constantinople switch block (nil = no fork, 0 = already activated) // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` @@ -152,7 +153,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Engine: %v}", c.ChainId, c.HomesteadBlock, c.DAOForkBlock, @@ -161,6 +162,7 @@ func (c *ChainConfig) String() string { c.EIP155Block, c.EIP158Block, c.ByzantiumBlock, + c.ConstantinopleBlock, engine, ) } @@ -191,6 +193,10 @@ func (c *ChainConfig) IsByzantium(num *big.Int) bool { return isForked(c.ByzantiumBlock, num) } +func (c *ChainConfig) IsConstantinople(num *big.Int) bool { + return isForked(c.ConstantinopleBlock, num) +} + // GasTable returns the gas table corresponding to the current phase (homestead or homestead reprice). // // The returned GasTable's fields shouldn't, under any circumstances, be changed. @@ -251,6 +257,9 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.ByzantiumBlock, newcfg.ByzantiumBlock, head) { return newCompatError("Byzantium fork block", c.ByzantiumBlock, newcfg.ByzantiumBlock) } + if isForkIncompatible(c.ConstantinopleBlock, newcfg.ConstantinopleBlock, head) { + return newCompatError("Constantinople fork block", c.ConstantinopleBlock, newcfg.ConstantinopleBlock) + } return nil } diff --git a/rpc/server.go b/rpc/server.go index 30c288349e..11373b504c 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -421,7 +421,7 @@ func (s *Server) readRequest(codec ServerCodec) ([]*serverRequest, bool, Error) } } } else { - requests[i] = &serverRequest{id: r.id, err: &methodNotFoundError{r.method, r.method}} + requests[i] = &serverRequest{id: r.id, err: &methodNotFoundError{r.service, r.method}} } continue } diff --git a/swarm/api/api.go b/swarm/api/api.go index 547c723d5a..73fc6e07a3 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -20,7 +20,9 @@ import ( "context" "fmt" "io" + "math/big" "net/http" + "path" "regexp" "strings" @@ -30,7 +32,10 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/contracts/ens" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -48,10 +53,161 @@ func (e *ErrResourceReturn) Key() string { return e.key } +var ( + apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil) + apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil) + apiPutCount = metrics.NewRegisteredCounter("api.put.count", nil) + apiPutFail = metrics.NewRegisteredCounter("api.put.fail", nil) + apiGetCount = metrics.NewRegisteredCounter("api.get.count", nil) + apiGetNotFound = metrics.NewRegisteredCounter("api.get.notfound", nil) + apiGetHttp300 = metrics.NewRegisteredCounter("api.get.http.300", nil) + apiModifyCount = metrics.NewRegisteredCounter("api.modify.count", nil) + apiModifyFail = metrics.NewRegisteredCounter("api.modify.fail", nil) + apiAddFileCount = metrics.NewRegisteredCounter("api.addfile.count", nil) + apiAddFileFail = metrics.NewRegisteredCounter("api.addfile.fail", nil) + apiRmFileCount = metrics.NewRegisteredCounter("api.removefile.count", nil) + apiRmFileFail = metrics.NewRegisteredCounter("api.removefile.fail", nil) + apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil) + apiAppendFileFail = metrics.NewRegisteredCounter("api.appendfile.fail", nil) +) + type Resolver interface { Resolve(string) (common.Hash, error) } +type ResolveValidator interface { + Resolver + Owner(node [32]byte) (common.Address, error) + HeaderByNumber(context.Context, *big.Int) (*types.Header, error) +} + +// NoResolverError is returned by MultiResolver.Resolve if no resolver +// can be found for the address. +type NoResolverError struct { + TLD string +} + +func NewNoResolverError(tld string) *NoResolverError { + return &NoResolverError{TLD: tld} +} + +func (e *NoResolverError) Error() string { + if e.TLD == "" { + return "no ENS resolver" + } + return fmt.Sprintf("no ENS endpoint configured to resolve .%s TLD names", e.TLD) +} + +// MultiResolver is used to resolve URL addresses based on their TLDs. +// Each TLD can have multiple resolvers, and the resoluton from the +// first one in the sequence will be returned. +type MultiResolver struct { + resolvers map[string][]ResolveValidator + nameHash func(string) common.Hash +} + +// MultiResolverOption sets options for MultiResolver and is used as +// arguments for its constructor. +type MultiResolverOption func(*MultiResolver) + +// MultiResolverOptionWithResolver adds a Resolver to a list of resolvers +// for a specific TLD. If TLD is an empty string, the resolver will be added +// to the list of default resolver, the ones that will be used for resolution +// of addresses which do not have their TLD resolver specified. +func MultiResolverOptionWithResolver(r ResolveValidator, tld string) MultiResolverOption { + return func(m *MultiResolver) { + m.resolvers[tld] = append(m.resolvers[tld], r) + } +} + +func MultiResolverOptionWithNameHash(nameHash func(string) common.Hash) MultiResolverOption { + return func(m *MultiResolver) { + m.nameHash = nameHash + } +} + +// NewMultiResolver creates a new instance of MultiResolver. +func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { + m = &MultiResolver{ + resolvers: make(map[string][]ResolveValidator), + nameHash: ens.EnsNode, + } + for _, o := range opts { + o(m) + } + return m +} + +// Resolve resolves address by choosing a Resolver by TLD. +// If there are more default Resolvers, or for a specific TLD, +// the Hash from the the first one which does not return error +// will be returned. +func (m *MultiResolver) Resolve(addr string) (h common.Hash, err error) { + rs, err := m.getResolveValidator(addr) + if err != nil { + return h, err + } + for _, r := range rs { + h, err = r.Resolve(addr) + if err == nil { + return + } + } + return +} + +func (m *MultiResolver) ValidateOwner(name string, address common.Address) (bool, error) { + rs, err := m.getResolveValidator(name) + if err != nil { + return false, err + } + var addr common.Address + for _, r := range rs { + addr, err = r.Owner(m.nameHash(name)) + // we hide the error if it is not for the last resolver we check + if err == nil { + return addr == address, nil + } + } + return false, err +} + +func (m *MultiResolver) HeaderByNumber(ctx context.Context, name string, blockNr *big.Int) (*types.Header, error) { + rs, err := m.getResolveValidator(name) + if err != nil { + return nil, err + } + for _, r := range rs { + var header *types.Header + header, err = r.HeaderByNumber(ctx, blockNr) + // we hide the error if it is not for the last resolver we check + if err == nil { + return header, nil + } + } + return nil, err +} + +func (m *MultiResolver) getResolveValidator(name string) ([]ResolveValidator, error) { + rs := m.resolvers[""] + tld := path.Ext(name) + if tld != "" { + tld = tld[1:] + rstld, ok := m.resolvers[tld] + if ok { + return rstld, nil + } + } + if len(rs) == 0 { + return rs, NewNoResolverError(tld) + } + return rs, nil +} + +func (m *MultiResolver) SetNameHash(nameHash func(string) common.Hash) { + m.nameHash = nameHash +} + /* Api implements webserver/file system related content storage and retrieval on top of the dpa @@ -93,6 +249,7 @@ type ErrResolve error // DNS Resolver func (self *Api) Resolve(uri *URI) (storage.Key, error) { + apiResolveCount.Inc(1) log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr)) // if the URI is immutable, check if the address is a hash @@ -107,6 +264,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { // if DNS is not configured, check if the address is a hash if self.dns == nil { if !isHash { + apiResolveFail.Inc(1) return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr) } return common.Hex2Bytes(uri.Addr), nil @@ -117,6 +275,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { if err == nil { return resolved[:], nil } else if !isHash { + apiResolveFail.Inc(1) return nil, err } return common.Hex2Bytes(uri.Addr), nil @@ -124,15 +283,18 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { // Put provides singleton manifest creation on top of dpa store func (self *Api) Put(content, contentType string) (k storage.Key, wait func(), err error) { + apiPutCount.Inc(1) r := strings.NewReader(content) key, waitContent, err := self.dpa.Store(r, int64(len(content))) if err != nil { + apiPutFail.Inc(1) return nil, nil, err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) r = strings.NewReader(manifest) key, waitManifest, err := self.dpa.Store(r, int64(len(manifest))) if err != nil { + apiPutFail.Inc(1) return nil, nil, err } return key, func() { @@ -145,8 +307,10 @@ func (self *Api) Put(content, contentType string) (k storage.Key, wait func(), e // to resolve basePath to content using dpa retrieve // it returns a section reader, mimeType, status and an error func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) { + apiGetCount.Inc(1) trie, err := loadManifest(self.dpa, key, nil) if err != nil { + apiGetNotFound.Inc(1) status = http.StatusNotFound log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err)) return @@ -172,6 +336,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe key = common.Hex2Bytes(entry.Hash) status = entry.Status if status == http.StatusMultipleChoices { + apiGetHttp300.Inc(1) return } else { mimeType = entry.ContentType @@ -180,6 +345,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe } } else { status = http.StatusNotFound + apiGetNotFound.Inc(1) err = fmt.Errorf("manifest entry for '%s' not found", path) log.Warn(fmt.Sprintf("%v", err)) } @@ -187,9 +353,11 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe } func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) { + apiModifyCount.Inc(1) quitC := make(chan bool) trie, err := loadManifest(self.dpa, key, quitC) if err != nil { + apiModifyFail.Inc(1) return nil, err } if contentHash != "" { @@ -204,19 +372,23 @@ func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) } if err := trie.recalcAndStore(); err != nil { + apiModifyFail.Inc(1) return nil, err } return trie.hash, nil } func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) { + apiAddFileCount.Inc(1) uri, err := Parse("bzz:/" + mhash) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } mkey, err := self.Resolve(uri) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } @@ -235,16 +407,19 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } fkey, err := mw.AddEntry(bytes.NewReader(content), entry) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } newMkey, err := mw.Store() if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } @@ -254,13 +429,16 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver } func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) { + apiRmFileCount.Inc(1) uri, err := Parse("bzz:/" + mhash) if err != nil { + apiRmFileFail.Inc(1) return "", err } mkey, err := self.Resolve(uri) if err != nil { + apiRmFileFail.Inc(1) return "", err } @@ -271,16 +449,19 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiRmFileFail.Inc(1) return "", err } err = mw.RemoveEntry(filepath.Join(path, fname)) if err != nil { + apiRmFileFail.Inc(1) return "", err } newMkey, err := mw.Store() if err != nil { + apiRmFileFail.Inc(1) return "", err } @@ -289,6 +470,7 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin } func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldKey storage.Key, offset int64, addSize int64, nameresolver bool) (storage.Key, string, error) { + apiAppendFileCount.Inc(1) buffSize := offset + addSize if buffSize < existingSize { @@ -317,10 +499,12 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte uri, err := Parse("bzz:/" + mhash) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } mkey, err := self.Resolve(uri) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -331,11 +515,13 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } err = mw.RemoveEntry(filepath.Join(path, fname)) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -349,11 +535,13 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte fkey, err := mw.AddEntry(io.Reader(combinedReader), entry) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } newMkey, err := mw.Store() if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -363,6 +551,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte } func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) { + uri, err := Parse("bzz:/" + mhash) if err != nil { return nil, nil, err diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 76b2751a58..49f12bb3a6 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -17,14 +17,17 @@ package api import ( + "context" "errors" "fmt" "io" "io/ioutil" + "math/big" "os" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -121,12 +124,12 @@ func TestApiPut(t *testing.T) { // testResolver implements the Resolver interface and either returns the given // hash if it is set, or returns a "name not found" error -type testResolver struct { +type testResolveValidator struct { hash *common.Hash } -func newTestResolver(addr string) *testResolver { - r := &testResolver{} +func newTestResolveValidator(addr string) *testResolveValidator { + r := &testResolveValidator{} if addr != "" { hash := common.HexToHash(addr) r.hash = &hash @@ -134,21 +137,28 @@ func newTestResolver(addr string) *testResolver { return r } -func (t *testResolver) Resolve(addr string) (common.Hash, error) { +func (t *testResolveValidator) Resolve(addr string) (common.Hash, error) { if t.hash == nil { return common.Hash{}, fmt.Errorf("DNS name not found: %q", addr) } return *t.hash, nil } +func (t *testResolveValidator) Owner(node [32]byte) (addr common.Address, err error) { + return +} +func (t *testResolveValidator) HeaderByNumber(context.Context, *big.Int) (header *types.Header, err error) { + return +} + // TestAPIResolve tests resolving URIs which can either contain content hashes // or ENS names func TestAPIResolve(t *testing.T) { ensAddr := "swarm.eth" hashAddr := "1111111111111111111111111111111111111111111111111111111111111111" resolvedAddr := "2222222222222222222222222222222222222222222222222222222222222222" - doesResolve := newTestResolver(resolvedAddr) - doesntResolve := newTestResolver("") + doesResolve := newTestResolveValidator(resolvedAddr) + doesntResolve := newTestResolveValidator("") type test struct { desc string @@ -237,3 +247,128 @@ func TestAPIResolve(t *testing.T) { }) } } + +func TestMultiResolver(t *testing.T) { + doesntResolve := newTestResolveValidator("") + + ethAddr := "swarm.eth" + ethHash := "0x2222222222222222222222222222222222222222222222222222222222222222" + ethResolve := newTestResolveValidator(ethHash) + + testAddr := "swarm.test" + testHash := "0x1111111111111111111111111111111111111111111111111111111111111111" + testResolve := newTestResolveValidator(testHash) + + tests := []struct { + desc string + r Resolver + addr string + result string + err error + }{ + { + desc: "No resolvers, returns error", + r: NewMultiResolver(), + err: NewNoResolverError(""), + }, + { + desc: "One default resolver, returns resolved address", + r: NewMultiResolver(MultiResolverOptionWithResolver(ethResolve, "")), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Two default resolvers, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, ""), + MultiResolverOptionWithResolver(ethResolve, ""), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Two default resolvers, first doesn't resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, ""), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Default resolver doesn't resolve, tld resolver resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Three TLD resolvers, third resolves, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, "eth"), + MultiResolverOptionWithResolver(doesntResolve, "eth"), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "One TLD resolver doesn't resolve, returns error", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "One defautl and one TLD resolver, all doesn't resolve, returns error", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(doesntResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + err: errors.New(`DNS name not found: "swarm.eth"`), + }, + { + desc: "Two TLD resolvers, both resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, "eth"), + MultiResolverOptionWithResolver(testResolve, "test"), + ), + addr: testAddr, + result: testHash, + }, + { + desc: "One TLD resolver, no default resolver, returns error for different TLD", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: testAddr, + err: NewNoResolverError("test"), + }, + } + for _, x := range tests { + t.Run(x.desc, func(t *testing.T) { + res, err := x.r.Resolve(x.addr) + if err == nil { + if x.err != nil { + t.Fatalf("expected error %q, got result %q", x.err, res.Hex()) + } + if res.Hex() != x.result { + t.Fatalf("expected result %q, got %q", x.result, res.Hex()) + } + } else { + if x.err == nil { + t.Fatalf("expected no error, got %q", err) + } + if err.Error() != x.err.Error() { + t.Fatalf("expected error %q, got %q", x.err, err) + } + } + }) + } +} diff --git a/swarm/api/config.go b/swarm/api/config.go index 14906176e9..6bb4d435d7 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -48,7 +48,7 @@ type Config struct { //*network.SyncParams Contract common.Address EnsRoot common.Address - EnsApi string + EnsAPIs []string Path string ListenAddr string Port string @@ -78,7 +78,7 @@ func NewConfig() (self *Config) { ListenAddr: DefaultHTTPListenAddr, Port: DefaultHTTPPort, Path: node.DefaultDataDir(), - EnsApi: node.DefaultIPCEndpoint("geth"), + EnsAPIs: nil, EnsRoot: ens.TestNetAddress, NetworkId: network.NetworkID, SwapEnabled: false, diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go index 0bee09324d..c932d0c784 100644 --- a/swarm/api/http/error.go +++ b/swarm/api/http/error.go @@ -29,12 +29,19 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" ) //templateMap holds a mapping of an HTTP error code to a template var templateMap map[int]*template.Template +//metrics variables +var ( + htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil) + jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil) +) + //parameters needed for formatting the correct HTML page type ResponseParams struct { Msg string @@ -132,6 +139,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { //return a HTML page func respondHtml(w http.ResponseWriter, params *ResponseParams) { + htmlCounter.Inc(1) err := params.template.Execute(w, params) if err != nil { log.Error(err.Error()) @@ -140,6 +148,7 @@ func respondHtml(w http.ResponseWriter, params *ResponseParams) { //return JSON func respondJson(w http.ResponseWriter, params *ResponseParams) { + jsonCounter.Inc(1) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(params) } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 029e9b5500..0acf86e40e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/pborman/uuid" @@ -50,6 +51,28 @@ type resourceResponse struct { Update storage.Key `json:"update"` } +var ( + postRawCount = metrics.NewRegisteredCounter("api.http.post.raw.count", nil) + postRawFail = metrics.NewRegisteredCounter("api.http.post.raw.fail", nil) + postFilesCount = metrics.NewRegisteredCounter("api.http.post.files.count", nil) + postFilesFail = metrics.NewRegisteredCounter("api.http.post.files.fail", nil) + deleteCount = metrics.NewRegisteredCounter("api.http.delete.count", nil) + deleteFail = metrics.NewRegisteredCounter("api.http.delete.fail", nil) + getCount = metrics.NewRegisteredCounter("api.http.get.count", nil) + getFail = metrics.NewRegisteredCounter("api.http.get.fail", nil) + getFileCount = metrics.NewRegisteredCounter("api.http.get.file.count", nil) + getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil) + getFileFail = metrics.NewRegisteredCounter("api.http.get.file.fail", nil) + getFilesCount = metrics.NewRegisteredCounter("api.http.get.files.count", nil) + getFilesFail = metrics.NewRegisteredCounter("api.http.get.files.fail", nil) + getListCount = metrics.NewRegisteredCounter("api.http.get.list.count", nil) + getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil) + requestCount = metrics.NewRegisteredCounter("http.request.count", nil) + htmlRequestCount = metrics.NewRegisteredCounter("http.request.html.count", nil) + jsonRequestCount = metrics.NewRegisteredCounter("http.request.json.count", nil) + requestTimer = metrics.NewRegisteredResettingTimer("http.request.time", nil) +) + // ServerConfig is the basic configuration needed for the HTTP server and also // includes CORS settings. type ServerConfig struct { @@ -98,18 +121,22 @@ type Request struct { // HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request // body in swarm and returns the resulting storage key as a text/plain response func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { + postRawCount.Inc(1) if r.uri.Path != "" { + postRawFail.Inc(1) Respond(w, r, "raw POST request cannot contain a path", http.StatusBadRequest) return } if r.Header.Get("Content-Length") == "" { + postRawFail.Inc(1) Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest) return } key, _, err := s.api.Store(r.Body, r.ContentLength) if err != nil { + postRawFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) return } @@ -126,8 +153,10 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { // existing manifest or to a new manifest under
and returns the // resulting manifest hash as a text/plain response func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { + postFilesCount.Inc(1) contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { + postFilesFail.Inc(1) Respond(w, r, err.Error(), http.StatusBadRequest) return } @@ -136,12 +165,14 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { if r.uri.Addr != "" { key, err = s.api.Resolve(r.uri) if err != nil { + postFilesFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) return } } else { key, err = s.api.NewManifest() if err != nil { + postFilesFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) return } @@ -161,6 +192,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { } }) if err != nil { + postFilesFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError) return } @@ -279,8 +311,10 @@ func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error //
from
and returns the resulting manifest hash as a // text/plain response func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { + deleteCount.Inc(1) key, err := s.api.Resolve(r.uri) if err != nil { + deleteFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) return } @@ -290,6 +324,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { return mw.RemoveEntry(r.uri.Path) }) if err != nil { + deleteFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot update manifest: %s", err), http.StatusInternalServerError) return } @@ -435,8 +470,10 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr // - bzz-hash://
and responds with the hash of the content stored // at the given storage key as a text/plain response func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { + getCount.Inc(1) key, err := s.api.Resolve(r.uri) if err != nil { + getFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) return } @@ -446,6 +483,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { walker, err := s.api.NewManifestWalker(key, nil) if err != nil { + getFail.Inc(1) Respond(w, r, fmt.Sprintf("%s is not a manifest", key), http.StatusBadRequest) return } @@ -474,6 +512,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { return api.SkipManifest }) if entry == nil { + getFail.Inc(1) Respond(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound) return } @@ -483,12 +522,13 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size reader := s.api.Retrieve(key) if _, err := reader.Size(nil); err != nil { + getFail.Inc(1) Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", key, err), http.StatusNotFound) return } switch { - case r.uri.Raw(): + case r.uri.Raw() || r.uri.DeprecatedRaw(): // allow the request to overwrite the content type using a query // parameter contentType := "application/octet-stream" @@ -508,19 +548,23 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // header of "application/x-tar" and returns a tar stream of all files // contained in the manifest func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { + getFilesCount.Inc(1) if r.uri.Path != "" { + getFilesFail.Inc(1) Respond(w, r, "files request cannot contain a path", http.StatusBadRequest) return } key, err := s.api.Resolve(r.uri) if err != nil { + getFilesFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) return } walker, err := s.api.NewManifestWalker(key, nil) if err != nil { + getFilesFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) return } @@ -568,6 +612,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { return nil }) if err != nil { + getFilesFail.Inc(1) log.Error(fmt.Sprintf("error generating tar stream: %s", err)) } } @@ -576,6 +621,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { // a list of all files contained in
under
grouped into // common prefixes using "/" as a delimiter func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { + getListCount.Inc(1) // ensure the root path has a trailing slash so that relative URLs work if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) @@ -584,6 +630,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { + getListFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) return } @@ -591,6 +638,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { list, err := s.getManifestList(key, r.uri.Path) if err != nil { + getListFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) return } @@ -608,6 +656,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { List: &list, }) if err != nil { + getListFail.Inc(1) log.Error(fmt.Sprintf("error rendering list HTML: %s", err)) } return @@ -676,6 +725,7 @@ func (s *Server) getManifestList(key storage.Key, prefix string) (list api.Manif // HandleGetFile handles a GET request to bzz://
/
and responds // with the content of the file at
from the given
func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { + getFileCount.Inc(1) // ensure the root path has a trailing slash so that relative URLs work if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) @@ -684,6 +734,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { + getFileFail.Inc(1) Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError) return } @@ -699,8 +750,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { } switch status { case http.StatusNotFound: + getFileNotFound.Inc(1) Respond(w, r, err.Error(), http.StatusNotFound) default: + getFileFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) } return @@ -712,6 +765,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { list, err := s.getManifestList(key, r.uri.Path) if err != nil { + getFileFail.Inc(1) Respond(w, r, err.Error(), http.StatusInternalServerError) return } @@ -724,6 +778,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size if _, err := reader.Size(nil); err != nil { + getFileNotFound.Inc(1) Respond(w, r, fmt.Sprintf("file not found %s: %s", r.uri, err), http.StatusNotFound) return } @@ -735,8 +790,18 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { req := &Request{Request: *r, ruid: uuid.New()[:8]} + requestCount.Inc(1) log.Info("serve request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI) + if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "text/html") { + + err := landingPageTemplate.Execute(w, nil) + if err != nil { + log.Error(fmt.Sprintf("error rendering landing page: %s", err)) + } + return + } + uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) if err != nil { Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest) diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go index 189a999122..cd9d212897 100644 --- a/swarm/api/http/templates.go +++ b/swarm/api/http/templates.go @@ -70,3 +70,146 @@ var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.Fu
+
+
+