Upgrade Geth

accounts/abi/bind/backends/simulated.go

@@ -428,10 +428,23 @@ func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumb
 	}
 	return fb.bc.GetHeaderByNumber(uint64(block.Int64())), nil
 }
+
 func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
 	return core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash)), nil
 }
 
+func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) {
+	receipts := core.GetBlockReceipts(fb.db, hash, core.GetBlockNumber(fb.db, hash))
+	if receipts == nil {
+		return nil, nil
+	}
+	logs := make([][]*types.Log, len(receipts))
+	for i, receipt := range receipts {
+		logs[i] = receipt.Logs
+	}
+	return logs, nil
+}
+
 func (fb *filterBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription {
 	return event.NewSubscription(func(quit <-chan struct{}) error {
 		<-quit

cmd/faucet/faucet.go

@@ -686,8 +686,6 @@ func authTwitter(url string) (string, string, common.Address, error) {
 	if len(parts) < 4 || parts[len(parts)-2] != "status" {
 		return "", "", common.Address{}, errors.New("Invalid Twitter status URL")
 	}
-	username := parts[len(parts)-3]
-
 	// Twitter's API isn't really friendly with direct links. Still, we don't
 	// want to do ask read permissions from users, so just load the public posts and
 	// scrape it for the Ethereum address and profile URL.
@@ -697,6 +695,13 @@ func authTwitter(url string) (string, string, common.Address, error) {
 	}
 	defer res.Body.Close()
 
+	// Resolve the username from the final redirect, no intermediate junk
+	parts = strings.Split(res.Request.URL.String(), "/")
+	if len(parts) < 4 || parts[len(parts)-2] != "status" {
+		return "", "", common.Address{}, errors.New("Invalid Twitter status URL")
+	}
+	username := parts[len(parts)-3]
+
 	body, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return "", "", common.Address{}, err

cmd/puppeth/module_dashboard.go

@@ -631,6 +631,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da
 		"Tangerine":        conf.Genesis.Config.EIP150Block,
 		"Spurious":         conf.Genesis.Config.EIP155Block,
 		"Byzantium":        conf.Genesis.Config.ByzantiumBlock,
+		"Constantinople":   conf.Genesis.Config.ConstantinopleBlock,
 	})
 	files[filepath.Join(workdir, "index.html")] = indexfile.Bytes()
 

cmd/puppeth/wizard_intro.go

@@ -59,15 +59,16 @@ func (w *wizard) run() {
 	fmt.Println()
 
 	// Make sure we have a good network name to work with	fmt.Println()
+	// Docker accepts hyphens in image names, but doesn't like it for container names
 	if w.network == "" {
-		fmt.Println("Please specify a network name to administer (no spaces, please)")
+		fmt.Println("Please specify a network name to administer (no spaces or hyphens, please)")
 		for {
 			w.network = w.readString()
-			if !strings.Contains(w.network, " ") {
+			if !strings.Contains(w.network, " ") && !strings.Contains(w.network, "-") {
 				fmt.Printf("\nSweet, you can set this via --network=%s next time!\n\n", w.network)
 				break
 			}
-			log.Error("I also like to live dangerously, still no spaces")
+			log.Error("I also like to live dangerously, still no spaces or hyphens")
 		}
 	}
 	log.Info("Administering Ethereum network", "name", w.network)

cmd/swarm/config.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"reflect"
 	"strconv"
+	"strings"
 	"unicode"
 
 	cli "gopkg.in/urfave/cli.v1"
@@ -97,10 +98,15 @@ func buildConfig(ctx *cli.Context) (config *bzzapi.Config, err error) {
 	config = bzzapi.NewDefaultConfig()
 	//first load settings from config file (if provided)
 	config, err = configFileOverride(config, ctx)
+	if err != nil {
+		return nil, err
+	}
 	//override settings provided by environment variables
 	config = envVarsOverride(config)
 	//override settings provided by command line
 	config = cmdLineOverride(config, ctx)
+	//validate configuration parameters
+	err = validateConfig(config)
 
 	return
 }
@@ -194,12 +200,16 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
 	}
 
-	//EnsApi can be set to "", so can't check for empty string, as it is allowed!
 	if ctx.GlobalIsSet(EnsAPIFlag.Name) {
-		currentConfig.EnsApi = ctx.GlobalString(EnsAPIFlag.Name)
+		ensAPIs := ctx.GlobalStringSlice(EnsAPIFlag.Name)
+		// preserve backward compatibility to disable ENS with --ens-api=""
+		if len(ensAPIs) == 1 && ensAPIs[0] == "" {
+			ensAPIs = nil
+		}
+		currentConfig.EnsAPIs = ensAPIs
 	}
 
-	if ensaddr := ctx.GlobalString(EnsAddrFlag.Name); ensaddr != "" {
+	if ensaddr := ctx.GlobalString(DeprecatedEnsAddrFlag.Name); ensaddr != "" {
 		currentConfig.EnsRoot = common.HexToAddress(ensaddr)
 	}
 
@@ -266,9 +276,8 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 		utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API)
 	}
 
-	//EnsApi can be set to "", so can't check for empty string, as it is allowed
+	if ensapi := os.Getenv(SWARM_ENV_ENS_API); ensapi != "" {
-	if ensapi, exists := os.LookupEnv(SWARM_ENV_ENS_API); exists {
+		currentConfig.EnsAPIs = strings.Split(ensapi, ",")
-		currentConfig.EnsApi = ensapi
 	}
 
 	if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" {
@@ -309,6 +318,43 @@ func checkDeprecated(ctx *cli.Context) {
 	if ctx.GlobalString(DeprecatedEthAPIFlag.Name) != "" {
 		utils.Fatalf("--ethapi is no longer a valid command line flag, please use --ens-api and/or --swap-api.")
 	}
+	// warn if --ens-api flag is set
+	if ctx.GlobalString(DeprecatedEnsAddrFlag.Name) != "" {
+		log.Warn("--ens-addr is no longer a valid command line flag, please use --ens-api to specify contract address.")
+	}
+}
+
+//validate configuration parameters
+func validateConfig(cfg *bzzapi.Config) (err error) {
+	for _, ensAPI := range cfg.EnsAPIs {
+		if ensAPI != "" {
+			if err := validateEnsAPIs(ensAPI); err != nil {
+				return fmt.Errorf("invalid format [tld:][contract-addr@]url for ENS API endpoint configuration %q: %v", ensAPI, err)
+			}
+		}
+	}
+	return nil
+}
+
+//validate EnsAPIs configuration parameter
+func validateEnsAPIs(s string) (err error) {
+	// missing contract address
+	if strings.HasPrefix(s, "@") {
+		return errors.New("missing contract address")
+	}
+	// missing url
+	if strings.HasSuffix(s, "@") {
+		return errors.New("missing url")
+	}
+	// missing tld
+	if strings.HasPrefix(s, ":") {
+		return errors.New("missing tld")
+	}
+	// missing url
+	if strings.HasSuffix(s, ":") {
+		return errors.New("missing url")
+	}
+	return nil
 }
 
 //print a Config as string

cmd/swarm/config_test.go

@@ -457,3 +457,98 @@ func TestCmdLineOverridesFile(t *testing.T) {
 
 	node.Shutdown()
 }
+
+func TestValidateConfig(t *testing.T) {
+	for _, c := range []struct {
+		cfg *api.Config
+		err string
+	}{
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"/data/testnet/geth.ipc",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"http://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"ws://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"test:/data/testnet/geth.ipc",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"test:ws://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"test:314159265dD8dbb310642f98f50C066173C1259b@/data/testnet/geth.ipc",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"eth:314159265dD8dbb310642f98f50C066173C1259b@http://127.0.0.1:1234",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"eth:314159265dD8dbb310642f98f50C066173C1259b@ws://127.0.0.1:12344",
+			}},
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"eth:",
+			}},
+			err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"eth:\": missing url",
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"314159265dD8dbb310642f98f50C066173C1259b@",
+			}},
+			err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"314159265dD8dbb310642f98f50C066173C1259b@\": missing url",
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				":314159265dD8dbb310642f98f50C066173C1259",
+			}},
+			err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \":314159265dD8dbb310642f98f50C066173C1259\": missing tld",
+		},
+		{
+			cfg: &api.Config{EnsAPIs: []string{
+				"@/data/testnet/geth.ipc",
+			}},
+			err: "invalid format [tld:][contract-addr@]url for ENS API endpoint configuration \"@/data/testnet/geth.ipc\": missing contract address",
+		},
+	} {
+		err := validateConfig(c.cfg)
+		if c.err != "" && err.Error() != c.err {
+			t.Errorf("expected error %q, got %q", c.err, err)
+		}
+		if c.err == "" && err != nil {
+			t.Errorf("unexpected error %q", err)
+		}
+	}
+}

cmd/swarm/main.go

@@ -17,11 +17,9 @@
 package main
 
 import (
-	"context"
 	"crypto/ecdsa"
 	"fmt"
 	"io/ioutil"
-	"math/big"
 	"os"
 	"os/signal"
 	"runtime"
@@ -29,14 +27,12 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
-	"time"
 
 	"github.com/EthereumCommonwealth/go-callisto/accounts"
 	"github.com/EthereumCommonwealth/go-callisto/accounts/keystore"
 	"github.com/EthereumCommonwealth/go-callisto/cmd/utils"
 	"github.com/EthereumCommonwealth/go-callisto/common"
 	"github.com/EthereumCommonwealth/go-callisto/console"
-	"github.com/EthereumCommonwealth/go-callisto/contracts/ens"
 	"github.com/EthereumCommonwealth/go-callisto/crypto"
 	"github.com/EthereumCommonwealth/go-callisto/ethclient"
 	"github.com/EthereumCommonwealth/go-callisto/internal/debug"
@@ -45,9 +41,9 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/p2p"
 	"github.com/EthereumCommonwealth/go-callisto/p2p/discover"
 	"github.com/EthereumCommonwealth/go-callisto/params"
-	"github.com/EthereumCommonwealth/go-callisto/rpc"
 	"github.com/EthereumCommonwealth/go-callisto/swarm"
 	bzzapi "github.com/EthereumCommonwealth/go-callisto/swarm/api"
+	swarmmetrics "github.com/EthereumCommonwealth/go-callisto/swarm/metrics"
 
 	"gopkg.in/urfave/cli.v1"
 )
@@ -110,16 +106,11 @@ var (
 		Usage:  "Swarm Syncing enabled (default true)",
 		EnvVar: SWARM_ENV_SYNC_ENABLE,
 	}
-	EnsAPIFlag = cli.StringFlag{
+	EnsAPIFlag = cli.StringSliceFlag{
 		Name:   "ens-api",
-		Usage:  "URL of the Ethereum API provider to use for ENS record lookups",
+		Usage:  "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url",
 		EnvVar: SWARM_ENV_ENS_API,
 	}
-	EnsAddrFlag = cli.StringFlag{
-		Name:   "ens-addr",
-		Usage:  "ENS contract address (default is detected as testnet or mainnet using --ens-api)",
-		EnvVar: SWARM_ENV_ENS_ADDR,
-	}
 	SwarmApiFlag = cli.StringFlag{
 		Name:  "bzzapi",
 		Usage: "Swarm HTTP endpoint",
@@ -156,6 +147,10 @@ var (
 		Name:  "ethapi",
 		Usage: "DEPRECATED: please use --ens-api and --swap-api",
 	}
+	DeprecatedEnsAddrFlag = cli.StringFlag{
+		Name:  "ens-addr",
+		Usage: "DEPRECATED: ENS contract address, please use --ens-api with contract address according to its format",
+	}
 )
 
 //declare a few constant error messages, useful for later error check comparisons in test
@@ -343,7 +338,6 @@ DEPRECATED: use 'swarm db clean'.
 		// bzzd-specific flags
 		CorsStringFlag,
 		EnsAPIFlag,
-		EnsAddrFlag,
 		SwarmTomlConfigPathFlag,
 		SwarmConfigPathFlag,
 		SwarmSwapEnabledFlag,
@@ -363,11 +357,17 @@ DEPRECATED: use 'swarm db clean'.
 		SwarmUploadMimeType,
 		//deprecated flags
 		DeprecatedEthAPIFlag,
+		DeprecatedEnsAddrFlag,
 	}
 	app.Flags = append(app.Flags, debug.Flags...)
+	app.Flags = append(app.Flags, swarmmetrics.Flags...)
 	app.Before = func(ctx *cli.Context) error {
 		runtime.GOMAXPROCS(runtime.NumCPU())
-		return debug.Setup(ctx)
+		if err := debug.Setup(ctx); err != nil {
+			return err
+		}
+		swarmmetrics.Setup(ctx)
+		return nil
 	}
 	app.After = func(ctx *cli.Context) error {
 		debug.Exit()
@@ -448,38 +448,6 @@ func bzzd(ctx *cli.Context) error {
 	return nil
 }
 
-// detectEnsAddr determines the ENS contract address by getting both the
-// version and genesis hash using the client and matching them to either
-// mainnet or testnet addresses
-func detectEnsAddr(client *rpc.Client) (common.Address, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	var version string
-	if err := client.CallContext(ctx, &version, "net_version"); err != nil {
-		return common.Address{}, err
-	}
-
-	block, err := ethclient.NewClient(client).BlockByNumber(ctx, big.NewInt(0))
-	if err != nil {
-		return common.Address{}, err
-	}
-
-	switch {
-
-	case version == "1" && block.Hash() == params.MainnetGenesisHash:
-		log.Info("using Mainnet ENS contract address", "addr", ens.MainNetAddress)
-		return ens.MainNetAddress, nil
-
-	case version == "3" && block.Hash() == params.TestnetGenesisHash:
-		log.Info("using Testnet ENS contract address", "addr", ens.TestNetAddress)
-		return ens.TestNetAddress, nil
-
-	default:
-		return common.Address{}, fmt.Errorf("unknown version and genesis hash: %s %s", version, block.Hash())
-	}
-}
-
 func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.Node) {
 
 	//define the swarm service boot function
@@ -494,27 +462,7 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.
 			}
 		}
 
-		var ensClient *ethclient.Client
+		return swarm.NewSwarm(ctx, swapClient, bzzconfig)
-		if bzzconfig.EnsApi != "" {
-			log.Info("connecting to ENS API", "url", bzzconfig.EnsApi)
-			client, err := rpc.Dial(bzzconfig.EnsApi)
-			if err != nil {
-				return nil, fmt.Errorf("error connecting to ENS API %s: %s", bzzconfig.EnsApi, err)
-			}
-			ensClient = ethclient.NewClient(client)
-
-			//no ENS root address set yet
-			if bzzconfig.EnsRoot == (common.Address{}) {
-				ensAddr, err := detectEnsAddr(client)
-				if err == nil {
-					bzzconfig.EnsRoot = ensAddr
-				} else {
-					log.Warn(fmt.Sprintf("could not determine ENS contract address, using default %s", bzzconfig.EnsRoot), "err", err)
-				}
-			}
-		}
-
-		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors)
 	}
 	//register within the ethereum node
 	if err := stack.Register(boot); err != nil {

common/big.go

@@ -25,6 +25,6 @@ var (
 	Big3   = big.NewInt(3)
 	Big0   = big.NewInt(0)
 	Big32  = big.NewInt(32)
-	Big256 = big.NewInt(0xff)
+	Big256 = big.NewInt(256)
 	Big257 = big.NewInt(257)
 )

consensus/ethash/ethash.go

@@ -34,10 +34,10 @@ import (
 
 	"github.com/EthereumCommonwealth/go-callisto/consensus"
 	"github.com/EthereumCommonwealth/go-callisto/log"
+	"github.com/EthereumCommonwealth/go-callisto/metrics"
 	"github.com/EthereumCommonwealth/go-callisto/rpc"
 	mmap "github.com/edsrzf/mmap-go"
 	"github.com/hashicorp/golang-lru/simplelru"
-	metrics "github.com/rcrowley/go-metrics"
 )
 
 var ErrInvalidDumpMagic = errors.New("invalid dump magic")

core/blockchain.go

@@ -46,7 +46,7 @@ import (
 )
 
 var (
-	blockInsertTimer = metrics.NewTimer("chain/inserts")
+	blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil)
 
 	ErrNoGenesis = errors.New("Genesis not found in chain")
 )
@@ -648,22 +648,21 @@ func (bc *BlockChain) Stop() {
 	bc.wg.Wait()
 
 	// Ensure the state of a recent block is also stored to disk before exiting.
-	// It is fine if this state does not exist (fast start/stop cycle), but it is
+	// We're writing three different states to catch different restart scenarios:
-	// advisable to leave an N block gap from the head so 1) a restart loads up
+	//  - HEAD:     So we don't need to reprocess any blocks in the general case
-	// the last N blocks as sync assistance to remote nodes; 2) a restart during
+	//  - HEAD-1:   So we don't do large reorgs if our HEAD becomes an uncle
-	// a (small) reorg doesn't require deep reprocesses; 3) chain "repair" from
+	//  - HEAD-127: So we have a hard limit on the number of blocks reexecuted
-	// missing states are constantly tested.
-	//
-	// This may be tuned a bit on mainnet if its too annoying to reprocess the last
-	// N blocks.
 	if !bc.cacheConfig.Disabled {
 		triedb := bc.stateCache.TrieDB()
-		if number := bc.CurrentBlock().NumberU64(); number >= triesInMemory {
-			recent := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - triesInMemory + 1)
 
-			log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
+		for _, offset := range []uint64{0, 1, triesInMemory - 1} {
-			if err := triedb.Commit(recent.Root(), true); err != nil {
+			if number := bc.CurrentBlock().NumberU64(); number > offset {
-				log.Error("Failed to commit recent state trie", "err", err)
+				recent := bc.GetBlockByNumber(number - offset)
+
+				log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
+				if err := triedb.Commit(recent.Root(), true); err != nil {
+					log.Error("Failed to commit recent state trie", "err", err)
+				}
 			}
 		}
 		for !bc.triegc.Empty() {

core/blockchain_test.go

@@ -34,26 +34,6 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/params"
 )
 
-// newTestBlockChain creates a blockchain without validation.
-func newTestBlockChain(fake bool) *BlockChain {
-	db, _ := ethdb.NewMemDatabase()
-	gspec := &Genesis{
-		Config:     params.TestChainConfig,
-		Difficulty: big.NewInt(1),
-	}
-	gspec.MustCommit(db)
-	engine := ethash.NewFullFaker()
-	if !fake {
-		engine = ethash.NewTester()
-	}
-	blockchain, err := NewBlockChain(db, nil, gspec.Config, engine, vm.Config{})
-	if err != nil {
-		panic(err)
-	}
-	blockchain.SetValidator(bproc{})
-	return blockchain
-}
-
 // Test fork of length N starting from block i
 func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int)) {
 	// Copy old chain up to #i into a new db
@@ -183,13 +163,18 @@ func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t *
 }
 
 func TestLastBlock(t *testing.T) {
-	bchain := newTestBlockChain(false)
+	_, blockchain, err := newCanonical(ethash.NewFaker(), 0, true)
-	defer bchain.Stop()
+	if err != nil {
+		t.Fatalf("failed to create pristine chain: %v", err)
+	}
+	defer blockchain.Stop()
 
-	block := makeBlockChain(bchain.CurrentBlock(), 1, ethash.NewFaker(), bchain.db, 0)[0]
+	blocks := makeBlockChain(blockchain.CurrentBlock(), 1, ethash.NewFullFaker(), blockchain.db, 0)
-	bchain.insert(block)
+	if _, err := blockchain.InsertChain(blocks); err != nil {
-	if block.Hash() != GetHeadBlockHash(bchain.db) {
+		t.Fatalf("Failed to insert block: %v", err)
-		t.Errorf("Write/Get HeadBlockHash failed")
+	}
+	if blocks[len(blocks)-1].Hash() != GetHeadBlockHash(blockchain.db) {
+		t.Fatalf("Write/Get HeadBlockHash failed")
 	}
 }
 
@@ -337,55 +322,13 @@ func testBrokenChain(t *testing.T, full bool) {
 	}
 }
 
-type bproc struct{}
-
-func (bproc) ValidateBody(*types.Block) error { return nil }
-func (bproc) ValidateState(block, parent *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64) error {
-	return nil
-}
-func (bproc) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (types.Receipts, []*types.Log, uint64, error) {
-	return nil, nil, 0, nil
-}
-
-func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Header {
-	blocks := makeBlockChainWithDiff(genesis, d, seed)
-	headers := make([]*types.Header, len(blocks))
-	for i, block := range blocks {
-		headers[i] = block.Header()
-	}
-	return headers
-}
-
-func makeBlockChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block {
-	var chain []*types.Block
-	for i, difficulty := range d {
-		header := &types.Header{
-			Coinbase:    common.Address{seed},
-			Number:      big.NewInt(int64(i + 1)),
-			Difficulty:  big.NewInt(int64(difficulty)),
-			UncleHash:   types.EmptyUncleHash,
-			TxHash:      types.EmptyRootHash,
-			ReceiptHash: types.EmptyRootHash,
-			Time:        big.NewInt(int64(i) + 1),
-		}
-		if i == 0 {
-			header.ParentHash = genesis.Hash()
-		} else {
-			header.ParentHash = chain[i-1].Hash()
-		}
-		block := types.NewBlockWithHeader(header)
-		chain = append(chain, block)
-	}
-	return chain
-}
-
 // Tests that reorganising a long difficult chain after a short easy one
 // overwrites the canonical numbers and links in the database.
 func TestReorgLongHeaders(t *testing.T) { testReorgLong(t, false) }
 func TestReorgLongBlocks(t *testing.T)  { testReorgLong(t, true) }
 
 func testReorgLong(t *testing.T, full bool) {
-	testReorg(t, []int{1, 2, 4}, []int{1, 2, 3, 4}, 10, full)
+	testReorg(t, []int64{0, 0, -9}, []int64{0, 0, 0, -9}, 393280, full)
 }
 
 // Tests that reorganising a short difficult chain after a long easy one
@@ -394,45 +337,82 @@ func TestReorgShortHeaders(t *testing.T) { testReorgShort(t, false) }
 func TestReorgShortBlocks(t *testing.T)  { testReorgShort(t, true) }
 
 func testReorgShort(t *testing.T, full bool) {
-	testReorg(t, []int{1, 2, 3, 4}, []int{1, 10}, 11, full)
+	// Create a long easy chain vs. a short heavy one. Due to difficulty adjustment
+	// we need a fairly long chain of blocks with different difficulties for a short
+	// one to become heavyer than a long one. The 96 is an empirical value.
+	easy := make([]int64, 96)
+	for i := 0; i < len(easy); i++ {
+		easy[i] = 60
+	}
+	diff := make([]int64, len(easy)-1)
+	for i := 0; i < len(diff); i++ {
+		diff[i] = -9
+	}
+	testReorg(t, easy, diff, 12615120, full)
 }
 
-func testReorg(t *testing.T, first, second []int, td int64, full bool) {
+func testReorg(t *testing.T, first, second []int64, td int64, full bool) {
-	bc := newTestBlockChain(true)
+	// Create a pristine chain and database
-	defer bc.Stop()
+	db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full)
+	if err != nil {
+		t.Fatalf("failed to create pristine chain: %v", err)
+	}
+	defer blockchain.Stop()
 
 	// Insert an easy and a difficult chain afterwards
+	easyBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(first), func(i int, b *BlockGen) {
+		b.OffsetTime(first[i])
+	})
+	diffBlocks, _ := GenerateChain(params.TestChainConfig, blockchain.CurrentBlock(), ethash.NewFaker(), db, len(second), func(i int, b *BlockGen) {
+		b.OffsetTime(second[i])
+	})
 	if full {
-		bc.InsertChain(makeBlockChainWithDiff(bc.genesisBlock, first, 11))
+		if _, err := blockchain.InsertChain(easyBlocks); err != nil {
-		bc.InsertChain(makeBlockChainWithDiff(bc.genesisBlock, second, 22))
+			t.Fatalf("failed to insert easy chain: %v", err)
+		}
+		if _, err := blockchain.InsertChain(diffBlocks); err != nil {
+			t.Fatalf("failed to insert difficult chain: %v", err)
+		}
 	} else {
-		bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, first, 11), 1)
+		easyHeaders := make([]*types.Header, len(easyBlocks))
-		bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, second, 22), 1)
+		for i, block := range easyBlocks {
+			easyHeaders[i] = block.Header()
+		}
+		diffHeaders := make([]*types.Header, len(diffBlocks))
+		for i, block := range diffBlocks {
+			diffHeaders[i] = block.Header()
+		}
+		if _, err := blockchain.InsertHeaderChain(easyHeaders, 1); err != nil {
+			t.Fatalf("failed to insert easy chain: %v", err)
+		}
+		if _, err := blockchain.InsertHeaderChain(diffHeaders, 1); err != nil {
+			t.Fatalf("failed to insert difficult chain: %v", err)
+		}
 	}
 	// Check that the chain is valid number and link wise
 	if full {
-		prev := bc.CurrentBlock()
+		prev := blockchain.CurrentBlock()
-		for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) {
+		for block := blockchain.GetBlockByNumber(blockchain.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, blockchain.GetBlockByNumber(block.NumberU64()-1) {
 			if prev.ParentHash() != block.Hash() {
 				t.Errorf("parent block hash mismatch: have %x, want %x", prev.ParentHash(), block.Hash())
 			}
 		}
 	} else {
-		prev := bc.CurrentHeader()
+		prev := blockchain.CurrentHeader()
-		for header := bc.GetHeaderByNumber(bc.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, bc.GetHeaderByNumber(header.Number.Uint64()-1) {
+		for header := blockchain.GetHeaderByNumber(blockchain.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, blockchain.GetHeaderByNumber(header.Number.Uint64()-1) {
 			if prev.ParentHash != header.Hash() {
 				t.Errorf("parent header hash mismatch: have %x, want %x", prev.ParentHash, header.Hash())
 			}
 		}
 	}
 	// Make sure the chain total difficulty is the correct one
-	want := new(big.Int).Add(bc.genesisBlock.Difficulty(), big.NewInt(td))
+	want := new(big.Int).Add(blockchain.genesisBlock.Difficulty(), big.NewInt(td))
 	if full {
-		if have := bc.GetTdByHash(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 {
+		if have := blockchain.GetTdByHash(blockchain.CurrentBlock().Hash()); have.Cmp(want) != 0 {
 			t.Errorf("total difficulty mismatch: have %v, want %v", have, want)
 		}
 	} else {
-		if have := bc.GetTdByHash(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 {
+		if have := blockchain.GetTdByHash(blockchain.CurrentHeader().Hash()); have.Cmp(want) != 0 {
 			t.Errorf("total difficulty mismatch: have %v, want %v", have, want)
 		}
 	}
@@ -443,19 +423,28 @@ func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false) }
 func TestBadBlockHashes(t *testing.T)  { testBadHashes(t, true) }
 
 func testBadHashes(t *testing.T, full bool) {
-	bc := newTestBlockChain(true)
+	// Create a pristine chain and database
-	defer bc.Stop()
+	db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full)
+	if err != nil {
+		t.Fatalf("failed to create pristine chain: %v", err)
+	}
+	defer blockchain.Stop()
 
 	// Create a chain, ban a hash and try to import
-	var err error
 	if full {
-		blocks := makeBlockChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10)
+		blocks := makeBlockChain(blockchain.CurrentBlock(), 3, ethash.NewFaker(), db, 10)
+
 		BadHashes[blocks[2].Header().Hash()] = true
-		_, err = bc.InsertChain(blocks)
+		defer func() { delete(BadHashes, blocks[2].Header().Hash()) }()
+
+		_, err = blockchain.InsertChain(blocks)
 	} else {
-		headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10)
+		headers := makeHeaderChain(blockchain.CurrentHeader(), 3, ethash.NewFaker(), db, 10)
+
 		BadHashes[headers[2].Hash()] = true
-		_, err = bc.InsertHeaderChain(headers, 1)
+		defer func() { delete(BadHashes, headers[2].Hash()) }()
+
+		_, err = blockchain.InsertHeaderChain(headers, 1)
 	}
 	if err != ErrBlacklistedHash {
 		t.Errorf("error mismatch: have: %v, want: %v", err, ErrBlacklistedHash)
@@ -468,40 +457,41 @@ func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false) }
 func TestReorgBadBlockHashes(t *testing.T)  { testReorgBadHashes(t, true) }
 
 func testReorgBadHashes(t *testing.T, full bool) {
-	bc := newTestBlockChain(true)
+	// Create a pristine chain and database
-	defer bc.Stop()
+	db, blockchain, err := newCanonical(ethash.NewFaker(), 0, full)
-
+	if err != nil {
+		t.Fatalf("failed to create pristine chain: %v", err)
+	}
 	// Create a chain, import and ban afterwards
-	headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10)
+	headers := makeHeaderChain(blockchain.CurrentHeader(), 4, ethash.NewFaker(), db, 10)
-	blocks := makeBlockChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10)
+	blocks := makeBlockChain(blockchain.CurrentBlock(), 4, ethash.NewFaker(), db, 10)
 
 	if full {
-		if _, err := bc.InsertChain(blocks); err != nil {
+		if _, err = blockchain.InsertChain(blocks); err != nil {
-			t.Fatalf("failed to import blocks: %v", err)
+			t.Errorf("failed to import blocks: %v", err)
 		}
-		if bc.CurrentBlock().Hash() != blocks[3].Hash() {
+		if blockchain.CurrentBlock().Hash() != blocks[3].Hash() {
-			t.Errorf("last block hash mismatch: have: %x, want %x", bc.CurrentBlock().Hash(), blocks[3].Header().Hash())
+			t.Errorf("last block hash mismatch: have: %x, want %x", blockchain.CurrentBlock().Hash(), blocks[3].Header().Hash())
 		}
 		BadHashes[blocks[3].Header().Hash()] = true
 		defer func() { delete(BadHashes, blocks[3].Header().Hash()) }()
 	} else {
-		if _, err := bc.InsertHeaderChain(headers, 1); err != nil {
+		if _, err = blockchain.InsertHeaderChain(headers, 1); err != nil {
-			t.Fatalf("failed to import headers: %v", err)
+			t.Errorf("failed to import headers: %v", err)
 		}
-		if bc.CurrentHeader().Hash() != headers[3].Hash() {
+		if blockchain.CurrentHeader().Hash() != headers[3].Hash() {
-			t.Errorf("last header hash mismatch: have: %x, want %x", bc.CurrentHeader().Hash(), headers[3].Hash())
+			t.Errorf("last header hash mismatch: have: %x, want %x", blockchain.CurrentHeader().Hash(), headers[3].Hash())
 		}
 		BadHashes[headers[3].Hash()] = true
 		defer func() { delete(BadHashes, headers[3].Hash()) }()
 	}
+	blockchain.Stop()
 
 	// Create a new BlockChain and check that it rolled back the state.
-	ncm, err := NewBlockChain(bc.db, nil, bc.chainConfig, ethash.NewFaker(), vm.Config{})
+	ncm, err := NewBlockChain(blockchain.db, nil, blockchain.chainConfig, ethash.NewFaker(), vm.Config{})
 	if err != nil {
 		t.Fatalf("failed to create new chain manager: %v", err)
 	}
-	defer ncm.Stop()
-
 	if full {
 		if ncm.CurrentBlock().Hash() != blocks[2].Header().Hash() {
 			t.Errorf("last block hash mismatch: have: %x, want %x", ncm.CurrentBlock().Hash(), blocks[2].Header().Hash())
@@ -514,6 +504,7 @@ func testReorgBadHashes(t *testing.T, full bool) {
 			t.Errorf("last header hash mismatch: have: %x, want %x", ncm.CurrentHeader().Hash(), headers[2].Hash())
 		}
 	}
+	ncm.Stop()
 }
 
 // Tests chain insertions in the face of one entity containing an invalid nonce.
@@ -989,10 +980,13 @@ done:
 
 // Tests if the canonical block can be fetched from the database during chain insertion.
 func TestCanonicalBlockRetrieval(t *testing.T) {
-	bc := newTestBlockChain(true)
+	_, blockchain, err := newCanonical(ethash.NewFaker(), 0, true)
-	defer bc.Stop()
+	if err != nil {
+		t.Fatalf("failed to create pristine chain: %v", err)
+	}
+	defer blockchain.Stop()
 
-	chain, _ := GenerateChain(bc.chainConfig, bc.genesisBlock, ethash.NewFaker(), bc.db, 10, func(i int, gen *BlockGen) {})
+	chain, _ := GenerateChain(blockchain.chainConfig, blockchain.genesisBlock, ethash.NewFaker(), blockchain.db, 10, func(i int, gen *BlockGen) {})
 
 	var pend sync.WaitGroup
 	pend.Add(len(chain))
@@ -1003,14 +997,14 @@ func TestCanonicalBlockRetrieval(t *testing.T) {
 
 			// try to retrieve a block by its canonical hash and see if the block data can be retrieved.
 			for {
-				ch := GetCanonicalHash(bc.db, block.NumberU64())
+				ch := GetCanonicalHash(blockchain.db, block.NumberU64())
 				if ch == (common.Hash{}) {
 					continue // busy wait for canonical hash to be written
 				}
 				if ch != block.Hash() {
 					t.Fatalf("unknown canonical hash, want %s, got %s", block.Hash().Hex(), ch.Hex())
 				}
-				fb := GetBlock(bc.db, ch, block.NumberU64())
+				fb := GetBlock(blockchain.db, ch, block.NumberU64())
 				if fb == nil {
 					t.Fatalf("unable to retrieve block %d for canonical hash: %s", block.NumberU64(), ch.Hex())
 				}
@@ -1021,7 +1015,7 @@ func TestCanonicalBlockRetrieval(t *testing.T) {
 			}
 		}(chain[i])
 
-		if _, err := bc.InsertChain(types.Blocks{chain[i]}); err != nil {
+		if _, err := blockchain.InsertChain(types.Blocks{chain[i]}); err != nil {
 			t.Fatalf("failed to insert block %d: %v", i, err)
 		}
 	}

core/database_util.go

@@ -70,8 +70,8 @@ var (
 
 	ErrChainConfigNotFound = errors.New("ChainConfig not found") // general config not found error
 
-	preimageCounter    = metrics.NewCounter("db/preimage/total")
+	preimageCounter    = metrics.NewRegisteredCounter("db/preimage/total", nil)
-	preimageHitCounter = metrics.NewCounter("db/preimage/hits")
+	preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
 )
 
 // TxLookupEntry is a positional metadata to help looking up the data content of

core/fees.go

@@ -1,23 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package core
-
-import (
-	"math/big"
-)
-
-var BlockReward = big.NewInt(5e+18)

core/genesis_test.go

@@ -118,10 +118,12 @@ func TestSetupGenesis(t *testing.T) {
 				// Commit the 'old' genesis block with Homestead transition at #2.
 				// Advance to block #4, past the homestead transition block of customg.
 				genesis := oldcustomg.MustCommit(db)
+
 				bc, _ := NewBlockChain(db, nil, oldcustomg.Config, ethash.NewFullFaker(), vm.Config{})
 				defer bc.Stop()
-				bc.SetValidator(bproc{})
+
-				bc.InsertChain(makeBlockChainWithDiff(genesis, []int{2, 3, 4, 5}, 0))
+				blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil)
+				bc.InsertChain(blocks)
 				bc.CurrentBlock()
 				// This should return a compatibility error.
 				return SetupGenesisBlock(db, &customg)

core/tx_pool.go

@@ -87,20 +87,20 @@ var (
 
 var (
 	// Metrics for the pending pool
-	pendingDiscardCounter   = metrics.NewCounter("txpool/pending/discard")
+	pendingDiscardCounter   = metrics.NewRegisteredCounter("txpool/pending/discard", nil)
-	pendingReplaceCounter   = metrics.NewCounter("txpool/pending/replace")
+	pendingReplaceCounter   = metrics.NewRegisteredCounter("txpool/pending/replace", nil)
-	pendingRateLimitCounter = metrics.NewCounter("txpool/pending/ratelimit") // Dropped due to rate limiting
+	pendingRateLimitCounter = metrics.NewRegisteredCounter("txpool/pending/ratelimit", nil) // Dropped due to rate limiting
-	pendingNofundsCounter   = metrics.NewCounter("txpool/pending/nofunds")   // Dropped due to out-of-funds
+	pendingNofundsCounter   = metrics.NewRegisteredCounter("txpool/pending/nofunds", nil)   // Dropped due to out-of-funds
 
 	// Metrics for the queued pool
-	queuedDiscardCounter   = metrics.NewCounter("txpool/queued/discard")
+	queuedDiscardCounter   = metrics.NewRegisteredCounter("txpool/queued/discard", nil)
-	queuedReplaceCounter   = metrics.NewCounter("txpool/queued/replace")
+	queuedReplaceCounter   = metrics.NewRegisteredCounter("txpool/queued/replace", nil)
-	queuedRateLimitCounter = metrics.NewCounter("txpool/queued/ratelimit") // Dropped due to rate limiting
+	queuedRateLimitCounter = metrics.NewRegisteredCounter("txpool/queued/ratelimit", nil) // Dropped due to rate limiting
-	queuedNofundsCounter   = metrics.NewCounter("txpool/queued/nofunds")   // Dropped due to out-of-funds
+	queuedNofundsCounter   = metrics.NewRegisteredCounter("txpool/queued/nofunds", nil)   // Dropped due to out-of-funds
 
 	// General tx metrics
-	invalidTxCounter     = metrics.NewCounter("txpool/invalid")
+	invalidTxCounter     = metrics.NewRegisteredCounter("txpool/invalid", nil)
-	underpricedTxCounter = metrics.NewCounter("txpool/underpriced")
+	underpricedTxCounter = metrics.NewRegisteredCounter("txpool/underpriced", nil)
 )
 
 // TxStatus is the current status of a transaction as seen by the pool.

core/vm/instructions.go

@@ -302,6 +302,66 @@ func opMulmod(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *S
 	return nil, nil
 }
 
+// opSHL implements Shift Left
+// The SHL instruction (shift left) pops 2 values from the stack, first arg1 and then arg2,
+// and pushes on the stack arg2 shifted to the left by arg1 number of bits.
+func opSHL(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
+	// Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards
+	shift, value := math.U256(stack.pop()), math.U256(stack.peek())
+	defer evm.interpreter.intPool.put(shift) // First operand back into the pool
+
+	if shift.Cmp(common.Big256) >= 0 {
+		value.SetUint64(0)
+		return nil, nil
+	}
+	n := uint(shift.Uint64())
+	math.U256(value.Lsh(value, n))
+
+	return nil, nil
+}
+
+// opSHR implements Logical Shift Right
+// The SHR instruction (logical shift right) pops 2 values from the stack, first arg1 and then arg2,
+// and pushes on the stack arg2 shifted to the right by arg1 number of bits with zero fill.
+func opSHR(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
+	// Note, second operand is left in the stack; accumulate result into it, and no need to push it afterwards
+	shift, value := math.U256(stack.pop()), math.U256(stack.peek())
+	defer evm.interpreter.intPool.put(shift) // First operand back into the pool
+
+	if shift.Cmp(common.Big256) >= 0 {
+		value.SetUint64(0)
+		return nil, nil
+	}
+	n := uint(shift.Uint64())
+	math.U256(value.Rsh(value, n))
+
+	return nil, nil
+}
+
+// opSAR implements Arithmetic Shift Right
+// The SAR instruction (arithmetic shift right) pops 2 values from the stack, first arg1 and then arg2,
+// and pushes on the stack arg2 shifted to the right by arg1 number of bits with sign extension.
+func opSAR(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
+	// Note, S256 returns (potentially) a new bigint, so we're popping, not peeking this one
+	shift, value := math.U256(stack.pop()), math.S256(stack.pop())
+	defer evm.interpreter.intPool.put(shift) // First operand back into the pool
+
+	if shift.Cmp(common.Big256) >= 0 {
+		if value.Sign() > 0 {
+			value.SetUint64(0)
+		} else {
+			value.SetInt64(-1)
+		}
+		stack.push(math.U256(value))
+		return nil, nil
+	}
+	n := uint(shift.Uint64())
+	value.Rsh(value, n)
+	stack.push(math.U256(value))
+
+	return nil, nil
+}
+
 func opSha3(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error) {
 	offset, size := stack.pop(), stack.pop()
 	data := memory.Get(offset.Int64(), size.Int64())

core/vm/instructions_test.go

@@ -24,6 +24,48 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/params"
 )
 
+type twoOperandTest struct {
+	x        string
+	y        string
+	expected string
+}
+
+func testTwoOperandOp(t *testing.T, tests []twoOperandTest, opFn func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error)) {
+	var (
+		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
+		stack = newstack()
+		pc    = uint64(0)
+	)
+	for i, test := range tests {
+		x := new(big.Int).SetBytes(common.Hex2Bytes(test.x))
+		shift := new(big.Int).SetBytes(common.Hex2Bytes(test.y))
+		expected := new(big.Int).SetBytes(common.Hex2Bytes(test.expected))
+		stack.push(x)
+		stack.push(shift)
+		opFn(&pc, env, nil, nil, stack)
+		actual := stack.pop()
+		if actual.Cmp(expected) != 0 {
+			t.Errorf("Testcase %d, expected  %v, got %v", i, expected, actual)
+		}
+		// Check pool usage
+		// 1.pool is not allowed to contain anything on the stack
+		// 2.pool is not allowed to contain the same pointers twice
+		if env.interpreter.intPool.pool.len() > 0 {
+
+			poolvals := make(map[*big.Int]struct{})
+			poolvals[actual] = struct{}{}
+
+			for env.interpreter.intPool.pool.len() > 0 {
+				key := env.interpreter.intPool.get()
+				if _, exist := poolvals[key]; exist {
+					t.Errorf("Testcase %d, pool contains double-entry", i)
+				}
+				poolvals[key] = struct{}{}
+			}
+		}
+	}
+}
+
 func TestByteOp(t *testing.T) {
 	var (
 		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
@@ -57,6 +99,98 @@ func TestByteOp(t *testing.T) {
 	}
 }
 
+func TestSHL(t *testing.T) {
+	// Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#shl-shift-left
+	tests := []twoOperandTest{
+		{"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000002"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "ff", "8000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "0100", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "0101", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "8000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe"},
+	}
+	testTwoOperandOp(t, tests, opSHL)
+}
+
+func TestSHR(t *testing.T) {
+	// Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#shr-logical-shift-right
+	tests := []twoOperandTest{
+		{"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "01", "4000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "ff", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "0100", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "0101", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"},
+	}
+	testTwoOperandOp(t, tests, opSHR)
+}
+
+func TestSAR(t *testing.T) {
+	// Testcases from https://github.com/ethereum/EIPs/blob/master/EIPS/eip-145.md#sar-arithmetic-shift-right
+	tests := []twoOperandTest{
+		{"0000000000000000000000000000000000000000000000000000000000000001", "00", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "01", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "01", "c000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "ff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "0100", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"8000000000000000000000000000000000000000000000000000000000000000", "0101", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "00", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "01", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"},
+		{"0000000000000000000000000000000000000000000000000000000000000000", "01", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"4000000000000000000000000000000000000000000000000000000000000000", "fe", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "f8", "000000000000000000000000000000000000000000000000000000000000007f"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "fe", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0100", "0000000000000000000000000000000000000000000000000000000000000000"},
+	}
+
+	testTwoOperandOp(t, tests, opSAR)
+}
+
+func TestSGT(t *testing.T) {
+	tests := []twoOperandTest{
+		{"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+	}
+	testTwoOperandOp(t, tests, opSgt)
+}
+
+func TestSLT(t *testing.T) {
+	tests := []twoOperandTest{
+		{"0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"0000000000000000000000000000000000000000000000000000000000000001", "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000001"},
+		{"8000000000000000000000000000000000000000000000000000000000000001", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"8000000000000000000000000000000000000000000000000000000000000001", "7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "0000000000000000000000000000000000000000000000000000000000000000"},
+		{"7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "8000000000000000000000000000000000000000000000000000000000000001", "0000000000000000000000000000000000000000000000000000000000000001"},
+	}
+	testTwoOperandOp(t, tests, opSlt)
+}
+
 func opBenchmark(bench *testing.B, op func(pc *uint64, evm *EVM, contract *Contract, memory *Memory, stack *Stack) ([]byte, error), args ...string) {
 	var (
 		env   = NewEVM(Context{}, nil, params.TestChainConfig, Config{EnableJit: false, ForceJit: false})
@@ -259,3 +393,22 @@ func BenchmarkOpMulmod(b *testing.B) {
 
 	opBenchmark(b, opMulmod, x, y, z)
 }
+
+func BenchmarkOpSHL(b *testing.B) {
+	x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff"
+	y := "ff"
+
+	opBenchmark(b, opSHL, x, y)
+}
+func BenchmarkOpSHR(b *testing.B) {
+	x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff"
+	y := "ff"
+
+	opBenchmark(b, opSHR, x, y)
+}
+func BenchmarkOpSAR(b *testing.B) {
+	x := "FBCDEF090807060504030201ffffffffFBCDEF090807060504030201ffffffff"
+	y := "ff"
+
+	opBenchmark(b, opSAR, x, y)
+}

core/vm/interpreter.go

@@ -68,6 +68,8 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter {
 	// we'll set the default jump table.
 	if !cfg.JumpTable[STOP].valid {
 		switch {
+		case evm.ChainConfig().IsConstantinople(evm.BlockNumber):
+			cfg.JumpTable = constantinopleInstructionSet
 		case evm.ChainConfig().IsByzantium(evm.BlockNumber):
 			cfg.JumpTable = byzantiumInstructionSet
 		case evm.ChainConfig().IsHomestead(evm.BlockNumber):

core/vm/jump_table.go

@@ -51,11 +51,38 @@ type operation struct {
 }
 
 var (
-	frontierInstructionSet  = NewFrontierInstructionSet()
+	frontierInstructionSet       = NewFrontierInstructionSet()
-	homesteadInstructionSet = NewHomesteadInstructionSet()
+	homesteadInstructionSet      = NewHomesteadInstructionSet()
-	byzantiumInstructionSet = NewByzantiumInstructionSet()
+	byzantiumInstructionSet      = NewByzantiumInstructionSet()
+	constantinopleInstructionSet = NewConstantinopleInstructionSet()
 )
 
+// NewConstantinopleInstructionSet returns the frontier, homestead
+// byzantium and contantinople instructions.
+func NewConstantinopleInstructionSet() [256]operation {
+	// instructions that can be executed during the byzantium phase.
+	instructionSet := NewByzantiumInstructionSet()
+	instructionSet[SHL] = operation{
+		execute:       opSHL,
+		gasCost:       constGasFunc(GasFastestStep),
+		validateStack: makeStackFunc(2, 1),
+		valid:         true,
+	}
+	instructionSet[SHR] = operation{
+		execute:       opSHR,
+		gasCost:       constGasFunc(GasFastestStep),
+		validateStack: makeStackFunc(2, 1),
+		valid:         true,
+	}
+	instructionSet[SAR] = operation{
+		execute:       opSAR,
+		gasCost:       constGasFunc(GasFastestStep),
+		validateStack: makeStackFunc(2, 1),
+		valid:         true,
+	}
+	return instructionSet
+}
+
 // NewByzantiumInstructionSet returns the frontier, homestead and
 // byzantium instructions.
 func NewByzantiumInstructionSet() [256]operation {

core/vm/opcodes.go

@@ -63,6 +63,9 @@ const (
 	XOR
 	NOT
 	BYTE
+	SHL
+	SHR
+	SAR
 
 	SHA3 = 0x20
 )
@@ -234,6 +237,9 @@ var opCodeToString = map[OpCode]string{
 	OR:     "OR",
 	XOR:    "XOR",
 	BYTE:   "BYTE",
+	SHL:    "SHL",
+	SHR:    "SHR",
+	SAR:    "SAR",
 	ADDMOD: "ADDMOD",
 	MULMOD: "MULMOD",
 
@@ -400,6 +406,9 @@ var stringToOp = map[string]OpCode{
 	"OR":             OR,
 	"XOR":            XOR,
 	"BYTE":           BYTE,
+	"SHL":            SHL,
+	"SHR":            SHR,
+	"SAR":            SAR,
 	"ADDMOD":         ADDMOD,
 	"MULMOD":         MULMOD,
 	"SHA3":           SHA3,

dashboard/dashboard.go

@@ -35,11 +35,11 @@ import (
 	"time"
 
 	"github.com/EthereumCommonwealth/go-callisto/log"
+	"github.com/EthereumCommonwealth/go-callisto/metrics"
 	"github.com/EthereumCommonwealth/go-callisto/p2p"
 	"github.com/EthereumCommonwealth/go-callisto/params"
 	"github.com/EthereumCommonwealth/go-callisto/rpc"
 	"github.com/elastic/gosigar"
-	"github.com/rcrowley/go-metrics"
 	"golang.org/x/net/websocket"
 )
 

eth/api_backend.go

@@ -104,6 +104,18 @@ func (b *EthApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash)
 	return core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash)), nil
 }
 
+func (b *EthApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) {
+	receipts := core.GetBlockReceipts(b.eth.chainDb, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash))
+	if receipts == nil {
+		return nil, nil
+	}
+	logs := make([][]*types.Log, len(receipts))
+	for i, receipt := range receipts {
+		logs[i] = receipt.Logs
+	}
+	return logs, nil
+}
+
 func (b *EthApiBackend) GetTd(blockHash common.Hash) *big.Int {
 	return b.eth.blockchain.GetTdByHash(blockHash)
 }

eth/downloader/downloader.go

@@ -31,8 +31,8 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/ethdb"
 	"github.com/EthereumCommonwealth/go-callisto/event"
 	"github.com/EthereumCommonwealth/go-callisto/log"
+	"github.com/EthereumCommonwealth/go-callisto/metrics"
 	"github.com/EthereumCommonwealth/go-callisto/params"
-	"github.com/rcrowley/go-metrics"
 )
 
 var (

eth/downloader/metrics.go

@@ -23,21 +23,21 @@ import (
 )
 
 var (
-	headerInMeter      = metrics.NewMeter("eth/downloader/headers/in")
+	headerInMeter      = metrics.NewRegisteredMeter("eth/downloader/headers/in", nil)
-	headerReqTimer     = metrics.NewTimer("eth/downloader/headers/req")
+	headerReqTimer     = metrics.NewRegisteredTimer("eth/downloader/headers/req", nil)
-	headerDropMeter    = metrics.NewMeter("eth/downloader/headers/drop")
+	headerDropMeter    = metrics.NewRegisteredMeter("eth/downloader/headers/drop", nil)
-	headerTimeoutMeter = metrics.NewMeter("eth/downloader/headers/timeout")
+	headerTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/headers/timeout", nil)
 
-	bodyInMeter      = metrics.NewMeter("eth/downloader/bodies/in")
+	bodyInMeter      = metrics.NewRegisteredMeter("eth/downloader/bodies/in", nil)
-	bodyReqTimer     = metrics.NewTimer("eth/downloader/bodies/req")
+	bodyReqTimer     = metrics.NewRegisteredTimer("eth/downloader/bodies/req", nil)
-	bodyDropMeter    = metrics.NewMeter("eth/downloader/bodies/drop")
+	bodyDropMeter    = metrics.NewRegisteredMeter("eth/downloader/bodies/drop", nil)
-	bodyTimeoutMeter = metrics.NewMeter("eth/downloader/bodies/timeout")
+	bodyTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/timeout", nil)
 
-	receiptInMeter      = metrics.NewMeter("eth/downloader/receipts/in")
+	receiptInMeter      = metrics.NewRegisteredMeter("eth/downloader/receipts/in", nil)
-	receiptReqTimer     = metrics.NewTimer("eth/downloader/receipts/req")
+	receiptReqTimer     = metrics.NewRegisteredTimer("eth/downloader/receipts/req", nil)
-	receiptDropMeter    = metrics.NewMeter("eth/downloader/receipts/drop")
+	receiptDropMeter    = metrics.NewRegisteredMeter("eth/downloader/receipts/drop", nil)
-	receiptTimeoutMeter = metrics.NewMeter("eth/downloader/receipts/timeout")
+	receiptTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/timeout", nil)
 
-	stateInMeter   = metrics.NewMeter("eth/downloader/states/in")
+	stateInMeter   = metrics.NewRegisteredMeter("eth/downloader/states/in", nil)
-	stateDropMeter = metrics.NewMeter("eth/downloader/states/drop")
+	stateDropMeter = metrics.NewRegisteredMeter("eth/downloader/states/drop", nil)
 )

eth/downloader/queue.go

@@ -28,7 +28,7 @@ import (
 	"github.com/EthereumCommonwealth/go-callisto/common"
 	"github.com/EthereumCommonwealth/go-callisto/core/types"
 	"github.com/EthereumCommonwealth/go-callisto/log"
-	"github.com/rcrowley/go-metrics"
+	"github.com/EthereumCommonwealth/go-callisto/metrics"
 	"gopkg.in/karalabe/cookiejar.v2/collections/prque"
 )
 

eth/fetcher/metrics.go

@@ -23,21 +23,21 @@ import (
 )
 
 var (
-	propAnnounceInMeter   = metrics.NewMeter("eth/fetcher/prop/announces/in")
+	propAnnounceInMeter   = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/in", nil)
-	propAnnounceOutTimer  = metrics.NewTimer("eth/fetcher/prop/announces/out")
+	propAnnounceOutTimer  = metrics.NewRegisteredTimer("eth/fetcher/prop/announces/out", nil)
-	propAnnounceDropMeter = metrics.NewMeter("eth/fetcher/prop/announces/drop")
+	propAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/drop", nil)
-	propAnnounceDOSMeter  = metrics.NewMeter("eth/fetcher/prop/announces/dos")
+	propAnnounceDOSMeter  = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/dos", nil)
 
-	propBroadcastInMeter   = metrics.NewMeter("eth/fetcher/prop/broadcasts/in")
+	propBroadcastInMeter   = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/in", nil)
-	propBroadcastOutTimer  = metrics.NewTimer("eth/fetcher/prop/broadcasts/out")
+	propBroadcastOutTimer  = metrics.NewRegisteredTimer("eth/fetcher/prop/broadcasts/out", nil)
-	propBroadcastDropMeter = metrics.NewMeter("eth/fetcher/prop/broadcasts/drop")
+	propBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/drop", nil)
-	propBroadcastDOSMeter  = metrics.NewMeter("eth/fetcher/prop/broadcasts/dos")
+	propBroadcastDOSMeter  = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/dos", nil)
 
-	headerFetchMeter = metrics.NewMeter("eth/fetcher/fetch/headers")
+	headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/headers", nil)
-	bodyFetchMeter   = metrics.NewMeter("eth/fetcher/fetch/bodies")
+	bodyFetchMeter   = metrics.NewRegisteredMeter("eth/fetcher/fetch/bodies", nil)
 
-	headerFilterInMeter  = metrics.NewMeter("eth/fetcher/filter/headers/in")
+	headerFilterInMeter  = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/in", nil)
-	headerFilterOutMeter = metrics.NewMeter("eth/fetcher/filter/headers/out")
+	headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/out", nil)
-	bodyFilterInMeter    = metrics.NewMeter("eth/fetcher/filter/bodies/in")
+	bodyFilterInMeter    = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/in", nil)
-	bodyFilterOutMeter   = metrics.NewMeter("eth/fetcher/filter/bodies/out")
+	bodyFilterOutMeter   = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/out", nil)
 )

eth/filters/filter.go

@@ -34,6 +34,7 @@ type Backend interface {
 	EventMux() *event.TypeMux
 	HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error)
 	GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error)
+	GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error)
 
 	SubscribeTxPreEvent(chan<- core.TxPreEvent) event.Subscription
 	SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
@@ -201,16 +202,28 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e
 // match the filter criteria. This function is called when the bloom filter signals a potential match.
 func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) {
 	// Get the logs of the block
-	receipts, err := f.backend.GetReceipts(ctx, header.Hash())
+	logsList, err := f.backend.GetLogs(ctx, header.Hash())
 	if err != nil {
 		return nil, err
 	}
 	var unfiltered []*types.Log
-	for _, receipt := range receipts {
+	for _, logs := range logsList {
-		unfiltered = append(unfiltered, receipt.Logs...)
+		unfiltered = append(unfiltered, logs...)
 	}
 	logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics)
 	if len(logs) > 0 {
+		// We have matching logs, check if we need to resolve full logs via the light client
+		if logs[0].TxHash == (common.Hash{}) {
+			receipts, err := f.backend.GetReceipts(ctx, header.Hash())
+			if err != nil {
+				return nil, err
+			}
+			unfiltered = unfiltered[:0]
+			for _, receipt := range receipts {
+				unfiltered = append(unfiltered, receipt.Logs...)
+			}
+			logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics)
+		}
 		return logs, nil
 	}
 	return nil, nil

eth/filters/filter_system.go

@@ -375,19 +375,35 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common.
 		// Get the logs of the block
 		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 		defer cancel()
-		receipts, err := es.backend.GetReceipts(ctx, header.Hash())
+		logsList, err := es.backend.GetLogs(ctx, header.Hash())
 		if err != nil {
 			return nil
 		}
 		var unfiltered []*types.Log
-		for _, receipt := range receipts {
+		for _, logs := range logsList {
-			for _, log := range receipt.Logs {
+			for _, log := range logs {
 				logcopy := *log
 				logcopy.Removed = remove
 				unfiltered = append(unfiltered, &logcopy)
 			}
 		}
 		logs := filterLogs(unfiltered, nil, nil, addresses, topics)
+		if len(logs) > 0 && logs[0].TxHash == (common.Hash{}) {
+			// We have matching but non-derived logs
+			receipts, err := es.backend.GetReceipts(ctx, header.Hash())
+			if err != nil {
+				return nil
+			}
+			unfiltered = unfiltered[:0]
+			for _, receipt := range receipts {
+				for _, log := range receipt.Logs {
+					logcopy := *log
+					logcopy.Removed = remove
+					unfiltered = append(unfiltered, &logcopy)
+				}
+			}
+			logs = filterLogs(unfiltered, nil, nil, addresses, topics)
+		}
 		return logs
 	}
 	return nil

eth/filters/filter_system_test.go

@@ -69,8 +69,19 @@ func (b *testBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumbe
 }
 
 func (b *testBackend) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) {
-	num := core.GetBlockNumber(b.db, blockHash)
+	number := core.GetBlockNumber(b.db, blockHash)
-	return core.GetBlockReceipts(b.db, blockHash, num), nil
+	return core.GetBlockReceipts(b.db, blockHash, number), nil
+}
+
+func (b *testBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) {
+	number := core.GetBlockNumber(b.db, blockHash)
+	receipts := core.GetBlockReceipts(b.db, blockHash, number)
+
+	logs := make([][]*types.Log, len(receipts))
+	for i, receipt := range receipts {
+		logs[i] = receipt.Logs
+	}
+	return logs, nil
 }
 
 func (b *testBackend) SubscribeTxPreEvent(ch chan<- core.TxPreEvent) event.Subscription {

eth/metrics.go

@@ -22,38 +22,38 @@ import (
 )
 
 var (
-	propTxnInPacketsMeter     = metrics.NewMeter("eth/prop/txns/in/packets")
+	propTxnInPacketsMeter     = metrics.NewRegisteredMeter("eth/prop/txns/in/packets", nil)
-	propTxnInTrafficMeter     = metrics.NewMeter("eth/prop/txns/in/traffic")
+	propTxnInTrafficMeter     = metrics.NewRegisteredMeter("eth/prop/txns/in/traffic", nil)
-	propTxnOutPacketsMeter    = metrics.NewMeter("eth/prop/txns/out/packets")
+	propTxnOutPacketsMeter    = metrics.NewRegisteredMeter("eth/prop/txns/out/packets", nil)
-	propTxnOutTrafficMeter    = metrics.NewMeter("eth/prop/txns/out/traffic")
+	propTxnOutTrafficMeter    = metrics.NewRegisteredMeter("eth/prop/txns/out/traffic", nil)
-	propHashInPacketsMeter    = metrics.NewMeter("eth/prop/hashes/in/packets")
+	propHashInPacketsMeter    = metrics.NewRegisteredMeter("eth/prop/hashes/in/packets", nil)
-	propHashInTrafficMeter    = metrics.NewMeter("eth/prop/hashes/in/traffic")
+	propHashInTrafficMeter    = metrics.NewRegisteredMeter("eth/prop/hashes/in/traffic", nil)
-	propHashOutPacketsMeter   = metrics.NewMeter("eth/prop/hashes/out/packets")
+	propHashOutPacketsMeter   = metrics.NewRegisteredMeter("eth/prop/hashes/out/packets", nil)
-	propHashOutTrafficMeter   = metrics.NewMeter("eth/prop/hashes/out/traffic")
+	propHashOutTrafficMeter   = metrics.NewRegisteredMeter("eth/prop/hashes/out/traffic", nil)
-	propBlockInPacketsMeter   = metrics.NewMeter("eth/prop/blocks/in/packets")
+	propBlockInPacketsMeter   = metrics.NewRegisteredMeter("eth/prop/blocks/in/packets", nil)
-	propBlockInTrafficMeter   = metrics.NewMeter("eth/prop/blocks/in/traffic")
+	propBlockInTrafficMeter   = metrics.NewRegisteredMeter("eth/prop/blocks/in/traffic", nil)
-	propBlockOutPacketsMeter  = metrics.NewMeter("eth/prop/blocks/out/packets")
+	propBlockOutPacketsMeter  = metrics.NewRegisteredMeter("eth/prop/blocks/out/packets", nil)
-	propBlockOutTrafficMeter  = metrics.NewMeter("eth/prop/blocks/out/traffic")
+	propBlockOutTrafficMeter  = metrics.NewRegisteredMeter("eth/prop/blocks/out/traffic", nil)
-	reqHeaderInPacketsMeter   = metrics.NewMeter("eth/req/headers/in/packets")
+	reqHeaderInPacketsMeter   = metrics.NewRegisteredMeter("eth/req/headers/in/packets", nil)
-	reqHeaderInTrafficMeter   = metrics.NewMeter("eth/req/headers/in/traffic")
+	reqHeaderInTrafficMeter   = metrics.NewRegisteredMeter("eth/req/headers/in/traffic", nil)
-	reqHeaderOutPacketsMeter  = metrics.NewMeter("eth/req/headers/out/packets")
+	reqHeaderOutPacketsMeter  = metrics.NewRegisteredMeter("eth/req/headers/out/packets", nil)
-	reqHeaderOutTrafficMeter  = metrics.NewMeter("eth/req/headers/out/traffic")
+	reqHeaderOutTrafficMeter  = metrics.NewRegisteredMeter("eth/req/headers/out/traffic", nil)
-	reqBodyInPacketsMeter     = metrics.NewMeter("eth/req/bodies/in/packets")
+	reqBodyInPacketsMeter     = metrics.NewRegisteredMeter("eth/req/bodies/in/packets", nil)
-	reqBodyInTrafficMeter     = metrics.NewMeter("eth/req/bodies/in/traffic")
+	reqBodyInTrafficMeter     = metrics.NewRegisteredMeter("eth/req/bodies/in/traffic", nil)
-	reqBodyOutPacketsMeter    = metrics.NewMeter("eth/req/bodies/out/packets")
+	reqBodyOutPacketsMeter    = metrics.NewRegisteredMeter("eth/req/bodies/out/packets", nil)
-	reqBodyOutTrafficMeter    = metrics.NewMeter("eth/req/bodies/out/traffic")
+	reqBodyOutTrafficMeter    = metrics.NewRegisteredMeter("eth/req/bodies/out/traffic", nil)
-	reqStateInPacketsMeter    = metrics.NewMeter("eth/req/states/in/packets")
+	reqStateInPacketsMeter    = metrics.NewRegisteredMeter("eth/req/states/in/packets", nil)
-	reqStateInTrafficMeter    = metrics.NewMeter("eth/req/states/in/traffic")
+	reqStateInTrafficMeter    = metrics.NewRegisteredMeter("eth/req/states/in/traffic", nil)
-	reqStateOutPacketsMeter   = metrics.NewMeter("eth/req/states/out/packets")
+	reqStateOutPacketsMeter   = metrics.NewRegisteredMeter("eth/req/states/out/packets", nil)
-	reqStateOutTrafficMeter   = metrics.NewMeter("eth/req/states/out/traffic")
+	reqStateOutTrafficMeter   = metrics.NewRegisteredMeter("eth/req/states/out/traffic", nil)
-	reqReceiptInPacketsMeter  = metrics.NewMeter("eth/req/receipts/in/packets")
+	reqReceiptInPacketsMeter  = metrics.NewRegisteredMeter("eth/req/receipts/in/packets", nil)
-	reqReceiptInTrafficMeter  = metrics.NewMeter("eth/req/receipts/in/traffic")
+	reqReceiptInTrafficMeter  = metrics.NewRegisteredMeter("eth/req/receipts/in/traffic", nil)
-	reqReceiptOutPacketsMeter = metrics.NewMeter("eth/req/receipts/out/packets")
+	reqReceiptOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/packets", nil)
-	reqReceiptOutTrafficMeter = metrics.NewMeter("eth/req/receipts/out/traffic")
+	reqReceiptOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/traffic", nil)
-	miscInPacketsMeter        = metrics.NewMeter("eth/misc/in/packets")
+	miscInPacketsMeter        = metrics.NewRegisteredMeter("eth/misc/in/packets", nil)
-	miscInTrafficMeter        = metrics.NewMeter("eth/misc/in/traffic")
+	miscInTrafficMeter        = metrics.NewRegisteredMeter("eth/misc/in/traffic", nil)
-	miscOutPacketsMeter       = metrics.NewMeter("eth/misc/out/packets")
+	miscOutPacketsMeter       = metrics.NewRegisteredMeter("eth/misc/out/packets", nil)
-	miscOutTrafficMeter       = metrics.NewMeter("eth/misc/out/traffic")
+	miscOutTrafficMeter       = metrics.NewRegisteredMeter("eth/misc/out/traffic", nil)
 )
 
 // meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of

ethdb/database.go

@@ -29,8 +29,6 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
-
-	gometrics "github.com/rcrowley/go-metrics"
 )
 
 var OpenFileLimit = 64
@@ -39,15 +37,15 @@ type LDBDatabase struct {
 	fn string      // filename for reporting
 	db *leveldb.DB // LevelDB instance
 
-	getTimer       gometrics.Timer // Timer for measuring the database get request counts and latencies
+	getTimer       metrics.Timer // Timer for measuring the database get request counts and latencies
-	putTimer       gometrics.Timer // Timer for measuring the database put request counts and latencies
+	putTimer       metrics.Timer // Timer for measuring the database put request counts and latencies
-	delTimer       gometrics.Timer // Timer for measuring the database delete request counts and latencies
+	delTimer       metrics.Timer // Timer for measuring the database delete request counts and latencies
-	missMeter      gometrics.Meter // Meter for measuring the missed database get requests
+	missMeter      metrics.Meter // Meter for measuring the missed database get requests
-	readMeter      gometrics.Meter // Meter for measuring the database get request data usage
+	readMeter      metrics.Meter // Meter for measuring the database get request data usage
-	writeMeter     gometrics.Meter // Meter for measuring the database put request data usage
+	writeMeter     metrics.Meter // Meter for measuring the database put request data usage
-	compTimeMeter  gometrics.Meter // Meter for measuring the total time spent in database compaction
+	compTimeMeter  metrics.Meter // Meter for measuring the total time spent in database compaction
-	compReadMeter  gometrics.Meter // Meter for measuring the data read during compaction
+	compReadMeter  metrics.Meter // Meter for measuring the data read during compaction
-	compWriteMeter gometrics.Meter // Meter for measuring the data written during compaction
+	compWriteMeter metrics.Meter // Meter for measuring the data written during compaction
 
 	quitLock sync.Mutex      // Mutex protecting the quit channel access
 	quitChan chan chan error // Quit channel to stop the metrics collection before closing the database
@@ -180,15 +178,15 @@ func (db *LDBDatabase) Meter(prefix string) {
 		return
 	}
 	// Initialize all the metrics collector at the requested prefix
-	db.getTimer = metrics.NewTimer(prefix + "user/gets")
+	db.getTimer = metrics.NewRegisteredTimer(prefix+"user/gets", nil)
-	db.putTimer = metrics.NewTimer(prefix + "user/puts")
+	db.putTimer = metrics.NewRegisteredTimer(prefix+"user/puts", nil)
-	db.delTimer = metrics.NewTimer(prefix + "user/dels")
+	db.delTimer = metrics.NewRegisteredTimer(prefix+"user/dels", nil)
-	db.missMeter = metrics.NewMeter(prefix + "user/misses")
+	db.missMeter = metrics.NewRegisteredMeter(prefix+"user/misses", nil)
-	db.readMeter = metrics.NewMeter(prefix + "user/reads")
+	db.readMeter = metrics.NewRegisteredMeter(prefix+"user/reads", nil)
-	db.writeMeter = metrics.NewMeter(prefix + "user/writes")
+	db.writeMeter = metrics.NewRegisteredMeter(prefix+"user/writes", nil)
-	db.compTimeMeter = metrics.NewMeter(prefix + "compact/time")
+	db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil)
-	db.compReadMeter = metrics.NewMeter(prefix + "compact/input")
+	db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil)
-	db.compWriteMeter = metrics.NewMeter(prefix + "compact/output")
+	db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil)
 
 	// Create a quit channel for the periodic collector and run it
 	db.quitLock.Lock()

internal/debug/flags.go

@@ -26,6 +26,8 @@ import (
 
 	"github.com/EthereumCommonwealth/go-callisto/log"
 	"github.com/EthereumCommonwealth/go-callisto/log/term"
+	"github.com/EthereumCommonwealth/go-callisto/metrics"
+	"github.com/EthereumCommonwealth/go-callisto/metrics/exp"
 	colorable "github.com/mattn/go-colorable"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -127,6 +129,10 @@ func Setup(ctx *cli.Context) error {
 
 	// pprof server
 	if ctx.GlobalBool(pprofFlag.Name) {
+		// Hook go-metrics into expvar on any /debug/metrics request, load all vars
+		// from the registry into expvar, and execute regular expvar handler.
+		exp.Exp(metrics.DefaultRegistry)
+
 		address := fmt.Sprintf("%s:%d", ctx.GlobalString(pprofAddrFlag.Name), ctx.GlobalInt(pprofPortFlag.Name))
 		go func() {
 			log.Info("Starting pprof server", "addr", fmt.Sprintf("http://%s/debug/pprof", address))

internal/ethapi/api.go

@@ -1032,17 +1032,20 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context,
 }
 
 // GetTransactionReceipt returns the transaction receipt for the given transaction hash.
-func (s *PublicTransactionPoolAPI) GetTransactionReceipt(hash common.Hash) (map[string]interface{}, error) {
+func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) {
 	tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash)
 	if tx == nil {
 		//return nil, errors.New("unknown transaction")
 		return nil, nil
 	}
-	receipt, _, _, _ := core.GetReceipt(s.b.ChainDb(), hash) // Old receipts don't have the lookup data available
+	receipts, err := s.b.GetReceipts(ctx, blockHash)
-	if receipt == nil {
+	if err != nil {
-	//	return nil, errors.New("unknown receipt")
+		return nil, err
-		return nil, nil
 	}
+	if len(receipts) <= int(index) {
+		return nil, errors.New("unknown receipt")
+	}
+	receipt := receipts[index]
 
 	var signer types.Signer = types.FrontierSigner{}
 	if tx.Protected() {

les/api_backend.go

@@ -87,6 +87,10 @@ func (b *LesApiBackend) GetReceipts(ctx context.Context, blockHash common.Hash)
 	return light.GetBlockReceipts(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash))
 }
 
+func (b *LesApiBackend) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) {
+	return light.GetBlockLogs(ctx, b.eth.odr, blockHash, core.GetBlockNumber(b.eth.chainDb, blockHash))
+}
+
 func (b *LesApiBackend) GetTd(blockHash common.Hash) *big.Int {
 	return b.eth.blockchain.GetTdByHash(blockHash)
 }

les/metrics.go

@@ -58,10 +58,10 @@ var (
 		reqReceiptInTrafficMeter  = metrics.NewMeter("eth/req/receipts/in/traffic")
 		reqReceiptOutPacketsMeter = metrics.NewMeter("eth/req/receipts/out/packets")
 		reqReceiptOutTrafficMeter = metrics.NewMeter("eth/req/receipts/out/traffic")*/
-	miscInPacketsMeter  = metrics.NewMeter("les/misc/in/packets")
+	miscInPacketsMeter  = metrics.NewRegisteredMeter("les/misc/in/packets", nil)
-	miscInTrafficMeter  = metrics.NewMeter("les/misc/in/traffic")
+	miscInTrafficMeter  = metrics.NewRegisteredMeter("les/misc/in/traffic", nil)
-	miscOutPacketsMeter = metrics.NewMeter("les/misc/out/packets")
+	miscOutPacketsMeter = metrics.NewRegisteredMeter("les/misc/out/packets", nil)
-	miscOutTrafficMeter = metrics.NewMeter("les/misc/out/traffic")
+	miscOutTrafficMeter = metrics.NewRegisteredMeter("les/misc/out/traffic", nil)
 )
 
 // meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of

light/odr_util.go

@@ -126,15 +126,48 @@ func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint
 // GetBlockReceipts retrieves the receipts generated by the transactions included
 // in a block given by its hash.
 func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (types.Receipts, error) {
+	// Retrieve the potentially incomplete receipts from disk or network
 	receipts := core.GetBlockReceipts(odr.Database(), hash, number)
-	if receipts != nil {
+	if receipts == nil {
-		return receipts, nil
+		r := &ReceiptsRequest{Hash: hash, Number: number}
+		if err := odr.Retrieve(ctx, r); err != nil {
+			return nil, err
+		}
+		receipts = r.Receipts
 	}
-	r := &ReceiptsRequest{Hash: hash, Number: number}
+	// If the receipts are incomplete, fill the derived fields
-	if err := odr.Retrieve(ctx, r); err != nil {
+	if len(receipts) > 0 && receipts[0].TxHash == (common.Hash{}) {
-		return nil, err
+		block, err := GetBlock(ctx, odr, hash, number)
+		if err != nil {
+			return nil, err
+		}
+		genesis := core.GetCanonicalHash(odr.Database(), 0)
+		config, _ := core.GetChainConfig(odr.Database(), genesis)
+
+		core.SetReceiptsData(config, block, receipts)
+		core.WriteBlockReceipts(odr.Database(), hash, number, receipts)
 	}
-	return r.Receipts, nil
+	return receipts, nil
+}
+
+// GetBlockLogs retrieves the logs generated by the transactions included in a
+// block given by its hash.
+func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) ([][]*types.Log, error) {
+	// Retrieve the potentially incomplete receipts from disk or network
+	receipts := core.GetBlockReceipts(odr.Database(), hash, number)
+	if receipts == nil {
+		r := &ReceiptsRequest{Hash: hash, Number: number}
+		if err := odr.Retrieve(ctx, r); err != nil {
+			return nil, err
+		}
+		receipts = r.Receipts
+	}
+	// Return the logs without deriving any computed fields on the receipts
+	logs := make([][]*types.Log, len(receipts))
+	for i, receipt := range receipts {
+		logs[i] = receipt.Logs
+	}
+	return logs, nil
 }
 
 // GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to the given bit index and section indexes

metrics/FORK.md

@@ -0,0 +1 @@
+This repo has been forked from https://github.com/rcrowley/go-metrics at commit e181e09

metrics/README.md

@@ -42,12 +42,22 @@ t.Update(47)
 Register() is not threadsafe. For threadsafe metric registration use
 GetOrRegister:
 
-```
+```go
 t := metrics.GetOrRegisterTimer("account.create.latency", nil)
 t.Time(func() {})
 t.Update(47)
 ```
 
+**NOTE:** Be sure to unregister short-lived meters and timers otherwise they will
+leak memory:
+
+```go
+// Will call Stop() on the Meter to allow for garbage collection
+metrics.Unregister("quux")
+// Or similarly for a Timer that embeds a Meter
+metrics.Unregister("bang")
+```
+
 Periodically log every metric in human-readable form to standard error:
 
 ```go
@@ -81,12 +91,13 @@ issues [#121](https://github.com/rcrowley/go-metrics/issues/121) and
 ```go
 import "github.com/vrischmann/go-metrics-influxdb"
 
-go influxdb.Influxdb(metrics.DefaultRegistry, 10e9, &influxdb.Config{
+go influxdb.InfluxDB(metrics.DefaultRegistry,
-    Host:     "127.0.0.1:8086",
+  10e9, 
-    Database: "metrics",
+  "127.0.0.1:8086", 
-    Username: "test",
+  "database-name", 
-    Password: "test",
+  "username", 
-})
+  "password"
+)
 ```
 
 Periodically upload every metric to Librato using the [Librato client](https://github.com/mihasya/go-metrics-librato):
@@ -146,8 +157,10 @@ Publishing Metrics
 
 Clients are available for the following destinations:
 
-* Librato - [https://github.com/mihasya/go-metrics-librato](https://github.com/mihasya/go-metrics-librato)
+* Librato - https://github.com/mihasya/go-metrics-librato
-* Graphite - [https://github.com/cyberdelia/go-metrics-graphite](https://github.com/cyberdelia/go-metrics-graphite)
+* Graphite - https://github.com/cyberdelia/go-metrics-graphite
-* InfluxDB - [https://github.com/vrischmann/go-metrics-influxdb](https://github.com/vrischmann/go-metrics-influxdb)
+* InfluxDB - https://github.com/vrischmann/go-metrics-influxdb
-* Ganglia - [https://github.com/appscode/metlia](https://github.com/appscode/metlia)
+* Ganglia - https://github.com/appscode/metlia
-* Prometheus - [https://github.com/deathowl/go-metrics-prometheus](https://github.com/deathowl/go-metrics-prometheus)
+* Prometheus - https://github.com/deathowl/go-metrics-prometheus
+* DataDog - https://github.com/syntaqx/go-metrics-datadog
+* SignalFX - https://github.com/pascallouisperez/go-metrics-signalfx

metrics/counter.go

@@ -22,7 +22,7 @@ func GetOrRegisterCounter(name string, r Registry) Counter {
 
 // NewCounter constructs a new StandardCounter.
 func NewCounter() Counter {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilCounter{}
 	}
 	return &StandardCounter{0}

metrics/counter_test.go

@@ -0,0 +1,77 @@
+package metrics
+
+import "testing"
+
+func BenchmarkCounter(b *testing.B) {
+	c := NewCounter()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		c.Inc(1)
+	}
+}
+
+func TestCounterClear(t *testing.T) {
+	c := NewCounter()
+	c.Inc(1)
+	c.Clear()
+	if count := c.Count(); 0 != count {
+		t.Errorf("c.Count(): 0 != %v\n", count)
+	}
+}
+
+func TestCounterDec1(t *testing.T) {
+	c := NewCounter()
+	c.Dec(1)
+	if count := c.Count(); -1 != count {
+		t.Errorf("c.Count(): -1 != %v\n", count)
+	}
+}
+
+func TestCounterDec2(t *testing.T) {
+	c := NewCounter()
+	c.Dec(2)
+	if count := c.Count(); -2 != count {
+		t.Errorf("c.Count(): -2 != %v\n", count)
+	}
+}
+
+func TestCounterInc1(t *testing.T) {
+	c := NewCounter()
+	c.Inc(1)
+	if count := c.Count(); 1 != count {
+		t.Errorf("c.Count(): 1 != %v\n", count)
+	}
+}
+
+func TestCounterInc2(t *testing.T) {
+	c := NewCounter()
+	c.Inc(2)
+	if count := c.Count(); 2 != count {
+		t.Errorf("c.Count(): 2 != %v\n", count)
+	}
+}
+
+func TestCounterSnapshot(t *testing.T) {
+	c := NewCounter()
+	c.Inc(1)
+	snapshot := c.Snapshot()
+	c.Inc(1)
+	if count := snapshot.Count(); 1 != count {
+		t.Errorf("c.Count(): 1 != %v\n", count)
+	}
+}
+
+func TestCounterZero(t *testing.T) {
+	c := NewCounter()
+	if count := c.Count(); 0 != count {
+		t.Errorf("c.Count(): 0 != %v\n", count)
+	}
+}
+
+func TestGetOrRegisterCounter(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredCounter("foo", r).Inc(47)
+	if c := GetOrRegisterCounter("foo", r); 47 != c.Count() {
+		t.Fatal(c)
+	}
+}

metrics/debug.go

@@ -22,7 +22,7 @@ var (
 // Capture new values for the Go garbage collector statistics exported in
 // debug.GCStats.  This is designed to be called as a goroutine.
 func CaptureDebugGCStats(r Registry, d time.Duration) {
-	for _ = range time.Tick(d) {
+	for range time.Tick(d) {
 		CaptureDebugGCStatsOnce(r)
 	}
 }
@@ -41,8 +41,8 @@ func CaptureDebugGCStatsOnce(r Registry) {
 	debug.ReadGCStats(&gcStats)
 	debugMetrics.ReadGCStats.UpdateSince(t)
 
-	debugMetrics.GCStats.LastGC.Update(int64(gcStats.LastGC.UnixNano()))
+	debugMetrics.GCStats.LastGC.Update(gcStats.LastGC.UnixNano())
-	debugMetrics.GCStats.NumGC.Update(int64(gcStats.NumGC))
+	debugMetrics.GCStats.NumGC.Update(gcStats.NumGC)
 	if lastGC != gcStats.LastGC && 0 < len(gcStats.Pause) {
 		debugMetrics.GCStats.Pause.Update(int64(gcStats.Pause[0]))
 	}

metrics/debug_test.go

@@ -0,0 +1,48 @@
+package metrics
+
+import (
+	"runtime"
+	"runtime/debug"
+	"testing"
+	"time"
+)
+
+func BenchmarkDebugGCStats(b *testing.B) {
+	r := NewRegistry()
+	RegisterDebugGCStats(r)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		CaptureDebugGCStatsOnce(r)
+	}
+}
+
+func TestDebugGCStatsBlocking(t *testing.T) {
+	if g := runtime.GOMAXPROCS(0); g < 2 {
+		t.Skipf("skipping TestDebugGCMemStatsBlocking with GOMAXPROCS=%d\n", g)
+		return
+	}
+	ch := make(chan int)
+	go testDebugGCStatsBlocking(ch)
+	var gcStats debug.GCStats
+	t0 := time.Now()
+	debug.ReadGCStats(&gcStats)
+	t1 := time.Now()
+	t.Log("i++ during debug.ReadGCStats:", <-ch)
+	go testDebugGCStatsBlocking(ch)
+	d := t1.Sub(t0)
+	t.Log(d)
+	time.Sleep(d)
+	t.Log("i++ during time.Sleep:", <-ch)
+}
+
+func testDebugGCStatsBlocking(ch chan int) {
+	i := 0
+	for {
+		select {
+		case ch <- i:
+			return
+		default:
+			i++
+		}
+	}
+}

metrics/ewma.go

@@ -17,7 +17,7 @@ type EWMA interface {
 
 // NewEWMA constructs a new EWMA with the given alpha.
 func NewEWMA(alpha float64) EWMA {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilEWMA{}
 	}
 	return &StandardEWMA{alpha: alpha}

metrics/ewma_test.go

@@ -0,0 +1,225 @@
+package metrics
+
+import "testing"
+
+func BenchmarkEWMA(b *testing.B) {
+	a := NewEWMA1()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		a.Update(1)
+		a.Tick()
+	}
+}
+
+func TestEWMA1(t *testing.T) {
+	a := NewEWMA1()
+	a.Update(3)
+	a.Tick()
+	if rate := a.Rate(); 0.6 != rate {
+		t.Errorf("initial a.Rate(): 0.6 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.22072766470286553 != rate {
+		t.Errorf("1 minute a.Rate(): 0.22072766470286553 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.08120116994196772 != rate {
+		t.Errorf("2 minute a.Rate(): 0.08120116994196772 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.029872241020718428 != rate {
+		t.Errorf("3 minute a.Rate(): 0.029872241020718428 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.01098938333324054 != rate {
+		t.Errorf("4 minute a.Rate(): 0.01098938333324054 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.004042768199451294 != rate {
+		t.Errorf("5 minute a.Rate(): 0.004042768199451294 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.0014872513059998212 != rate {
+		t.Errorf("6 minute a.Rate(): 0.0014872513059998212 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.0005471291793327122 != rate {
+		t.Errorf("7 minute a.Rate(): 0.0005471291793327122 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.00020127757674150815 != rate {
+		t.Errorf("8 minute a.Rate(): 0.00020127757674150815 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 7.404588245200814e-05 != rate {
+		t.Errorf("9 minute a.Rate(): 7.404588245200814e-05 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 2.7239957857491083e-05 != rate {
+		t.Errorf("10 minute a.Rate(): 2.7239957857491083e-05 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 1.0021020474147462e-05 != rate {
+		t.Errorf("11 minute a.Rate(): 1.0021020474147462e-05 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 3.6865274119969525e-06 != rate {
+		t.Errorf("12 minute a.Rate(): 3.6865274119969525e-06 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 1.3561976441886433e-06 != rate {
+		t.Errorf("13 minute a.Rate(): 1.3561976441886433e-06 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 4.989172314621449e-07 != rate {
+		t.Errorf("14 minute a.Rate(): 4.989172314621449e-07 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 1.8354139230109722e-07 != rate {
+		t.Errorf("15 minute a.Rate(): 1.8354139230109722e-07 != %v\n", rate)
+	}
+}
+
+func TestEWMA5(t *testing.T) {
+	a := NewEWMA5()
+	a.Update(3)
+	a.Tick()
+	if rate := a.Rate(); 0.6 != rate {
+		t.Errorf("initial a.Rate(): 0.6 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.49123845184678905 != rate {
+		t.Errorf("1 minute a.Rate(): 0.49123845184678905 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.4021920276213837 != rate {
+		t.Errorf("2 minute a.Rate(): 0.4021920276213837 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.32928698165641596 != rate {
+		t.Errorf("3 minute a.Rate(): 0.32928698165641596 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.269597378470333 != rate {
+		t.Errorf("4 minute a.Rate(): 0.269597378470333 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.2207276647028654 != rate {
+		t.Errorf("5 minute a.Rate(): 0.2207276647028654 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.18071652714732128 != rate {
+		t.Errorf("6 minute a.Rate(): 0.18071652714732128 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.14795817836496392 != rate {
+		t.Errorf("7 minute a.Rate(): 0.14795817836496392 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.12113791079679326 != rate {
+		t.Errorf("8 minute a.Rate(): 0.12113791079679326 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.09917933293295193 != rate {
+		t.Errorf("9 minute a.Rate(): 0.09917933293295193 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.08120116994196763 != rate {
+		t.Errorf("10 minute a.Rate(): 0.08120116994196763 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.06648189501740036 != rate {
+		t.Errorf("11 minute a.Rate(): 0.06648189501740036 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.05443077197364752 != rate {
+		t.Errorf("12 minute a.Rate(): 0.05443077197364752 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.04456414692860035 != rate {
+		t.Errorf("13 minute a.Rate(): 0.04456414692860035 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.03648603757513079 != rate {
+		t.Errorf("14 minute a.Rate(): 0.03648603757513079 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.0298722410207183831020718428 != rate {
+		t.Errorf("15 minute a.Rate(): 0.0298722410207183831020718428 != %v\n", rate)
+	}
+}
+
+func TestEWMA15(t *testing.T) {
+	a := NewEWMA15()
+	a.Update(3)
+	a.Tick()
+	if rate := a.Rate(); 0.6 != rate {
+		t.Errorf("initial a.Rate(): 0.6 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.5613041910189706 != rate {
+		t.Errorf("1 minute a.Rate(): 0.5613041910189706 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.5251039914257684 != rate {
+		t.Errorf("2 minute a.Rate(): 0.5251039914257684 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.4912384518467888184678905 != rate {
+		t.Errorf("3 minute a.Rate(): 0.4912384518467888184678905 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.459557003018789 != rate {
+		t.Errorf("4 minute a.Rate(): 0.459557003018789 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.4299187863442732 != rate {
+		t.Errorf("5 minute a.Rate(): 0.4299187863442732 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.4021920276213831 != rate {
+		t.Errorf("6 minute a.Rate(): 0.4021920276213831 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.37625345116383313 != rate {
+		t.Errorf("7 minute a.Rate(): 0.37625345116383313 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.3519877317060185 != rate {
+		t.Errorf("8 minute a.Rate(): 0.3519877317060185 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.3292869816564153165641596 != rate {
+		t.Errorf("9 minute a.Rate(): 0.3292869816564153165641596 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.3080502714195546 != rate {
+		t.Errorf("10 minute a.Rate(): 0.3080502714195546 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.2881831806538789 != rate {
+		t.Errorf("11 minute a.Rate(): 0.2881831806538789 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.26959737847033216 != rate {
+		t.Errorf("12 minute a.Rate(): 0.26959737847033216 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.2522102307052083 != rate {
+		t.Errorf("13 minute a.Rate(): 0.2522102307052083 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.23594443252115815 != rate {
+		t.Errorf("14 minute a.Rate(): 0.23594443252115815 != %v\n", rate)
+	}
+	elapseMinute(a)
+	if rate := a.Rate(); 0.2207276647028646247028654470286553 != rate {
+		t.Errorf("15 minute a.Rate(): 0.2207276647028646247028654470286553 != %v\n", rate)
+	}
+}
+
+func elapseMinute(a EWMA) {
+	for i := 0; i < 12; i++ {
+		a.Tick()
+	}
+}

metrics/exp/exp.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 	"sync"
 
-	"github.com/rcrowley/go-metrics"
+	"github.com/EthereumCommonwealth/go-callisto/metrics"
 )
 
 type exp struct {
@@ -97,22 +97,22 @@ func (exp *exp) publishHistogram(name string, metric metrics.Histogram) {
 	exp.getInt(name + ".count").Set(h.Count())
 	exp.getFloat(name + ".min").Set(float64(h.Min()))
 	exp.getFloat(name + ".max").Set(float64(h.Max()))
-	exp.getFloat(name + ".mean").Set(float64(h.Mean()))
+	exp.getFloat(name + ".mean").Set(h.Mean())
-	exp.getFloat(name + ".std-dev").Set(float64(h.StdDev()))
+	exp.getFloat(name + ".std-dev").Set(h.StdDev())
-	exp.getFloat(name + ".50-percentile").Set(float64(ps[0]))
+	exp.getFloat(name + ".50-percentile").Set(ps[0])
-	exp.getFloat(name + ".75-percentile").Set(float64(ps[1]))
+	exp.getFloat(name + ".75-percentile").Set(ps[1])
-	exp.getFloat(name + ".95-percentile").Set(float64(ps[2]))
+	exp.getFloat(name + ".95-percentile").Set(ps[2])
-	exp.getFloat(name + ".99-percentile").Set(float64(ps[3]))
+	exp.getFloat(name + ".99-percentile").Set(ps[3])
-	exp.getFloat(name + ".999-percentile").Set(float64(ps[4]))
+	exp.getFloat(name + ".999-percentile").Set(ps[4])
 }
 
 func (exp *exp) publishMeter(name string, metric metrics.Meter) {
 	m := metric.Snapshot()
 	exp.getInt(name + ".count").Set(m.Count())
-	exp.getFloat(name + ".one-minute").Set(float64(m.Rate1()))
+	exp.getFloat(name + ".one-minute").Set(m.Rate1())
-	exp.getFloat(name + ".five-minute").Set(float64(m.Rate5()))
+	exp.getFloat(name + ".five-minute").Set(m.Rate5())
-	exp.getFloat(name + ".fifteen-minute").Set(float64((m.Rate15())))
+	exp.getFloat(name + ".fifteen-minute").Set((m.Rate15()))
-	exp.getFloat(name + ".mean").Set(float64(m.RateMean()))
+	exp.getFloat(name + ".mean").Set(m.RateMean())
 }
 
 func (exp *exp) publishTimer(name string, metric metrics.Timer) {
@@ -121,17 +121,17 @@ func (exp *exp) publishTimer(name string, metric metrics.Timer) {
 	exp.getInt(name + ".count").Set(t.Count())
 	exp.getFloat(name + ".min").Set(float64(t.Min()))
 	exp.getFloat(name + ".max").Set(float64(t.Max()))
-	exp.getFloat(name + ".mean").Set(float64(t.Mean()))
+	exp.getFloat(name + ".mean").Set(t.Mean())
-	exp.getFloat(name + ".std-dev").Set(float64(t.StdDev()))
+	exp.getFloat(name + ".std-dev").Set(t.StdDev())
-	exp.getFloat(name + ".50-percentile").Set(float64(ps[0]))
+	exp.getFloat(name + ".50-percentile").Set(ps[0])
-	exp.getFloat(name + ".75-percentile").Set(float64(ps[1]))
+	exp.getFloat(name + ".75-percentile").Set(ps[1])
-	exp.getFloat(name + ".95-percentile").Set(float64(ps[2]))
+	exp.getFloat(name + ".95-percentile").Set(ps[2])
-	exp.getFloat(name + ".99-percentile").Set(float64(ps[3]))
+	exp.getFloat(name + ".99-percentile").Set(ps[3])
-	exp.getFloat(name + ".999-percentile").Set(float64(ps[4]))
+	exp.getFloat(name + ".999-percentile").Set(ps[4])
-	exp.getFloat(name + ".one-minute").Set(float64(t.Rate1()))
+	exp.getFloat(name + ".one-minute").Set(t.Rate1())
-	exp.getFloat(name + ".five-minute").Set(float64(t.Rate5()))
+	exp.getFloat(name + ".five-minute").Set(t.Rate5())
-	exp.getFloat(name + ".fifteen-minute").Set(float64((t.Rate15())))
+	exp.getFloat(name + ".fifteen-minute").Set(t.Rate15())
-	exp.getFloat(name + ".mean-rate").Set(float64(t.RateMean()))
+	exp.getFloat(name + ".mean-rate").Set(t.RateMean())
 }
 
 func (exp *exp) syncToExpvar() {

metrics/gauge.go

@@ -20,7 +20,7 @@ func GetOrRegisterGauge(name string, r Registry) Gauge {
 
 // NewGauge constructs a new StandardGauge.
 func NewGauge() Gauge {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilGauge{}
 	}
 	return &StandardGauge{0}
@@ -38,7 +38,7 @@ func NewRegisteredGauge(name string, r Registry) Gauge {
 
 // NewFunctionalGauge constructs a new FunctionalGauge.
 func NewFunctionalGauge(f func() int64) Gauge {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilGauge{}
 	}
 	return &FunctionalGauge{value: f}

metrics/gauge_float64.go

@@ -20,7 +20,7 @@ func GetOrRegisterGaugeFloat64(name string, r Registry) GaugeFloat64 {
 
 // NewGaugeFloat64 constructs a new StandardGaugeFloat64.
 func NewGaugeFloat64() GaugeFloat64 {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilGaugeFloat64{}
 	}
 	return &StandardGaugeFloat64{
@@ -40,7 +40,7 @@ func NewRegisteredGaugeFloat64(name string, r Registry) GaugeFloat64 {
 
 // NewFunctionalGauge constructs a new FunctionalGauge.
 func NewFunctionalGaugeFloat64(f func() float64) GaugeFloat64 {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilGaugeFloat64{}
 	}
 	return &FunctionalGaugeFloat64{value: f}

metrics/gauge_float64_test.go

@@ -0,0 +1,59 @@
+package metrics
+
+import "testing"
+
+func BenchmarkGuageFloat64(b *testing.B) {
+	g := NewGaugeFloat64()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		g.Update(float64(i))
+	}
+}
+
+func TestGaugeFloat64(t *testing.T) {
+	g := NewGaugeFloat64()
+	g.Update(float64(47.0))
+	if v := g.Value(); float64(47.0) != v {
+		t.Errorf("g.Value(): 47.0 != %v\n", v)
+	}
+}
+
+func TestGaugeFloat64Snapshot(t *testing.T) {
+	g := NewGaugeFloat64()
+	g.Update(float64(47.0))
+	snapshot := g.Snapshot()
+	g.Update(float64(0))
+	if v := snapshot.Value(); float64(47.0) != v {
+		t.Errorf("g.Value(): 47.0 != %v\n", v)
+	}
+}
+
+func TestGetOrRegisterGaugeFloat64(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredGaugeFloat64("foo", r).Update(float64(47.0))
+	t.Logf("registry: %v", r)
+	if g := GetOrRegisterGaugeFloat64("foo", r); float64(47.0) != g.Value() {
+		t.Fatal(g)
+	}
+}
+
+func TestFunctionalGaugeFloat64(t *testing.T) {
+	var counter float64
+	fg := NewFunctionalGaugeFloat64(func() float64 {
+		counter++
+		return counter
+	})
+	fg.Value()
+	fg.Value()
+	if counter != 2 {
+		t.Error("counter != 2")
+	}
+}
+
+func TestGetOrRegisterFunctionalGaugeFloat64(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredFunctionalGaugeFloat64("foo", r, func() float64 { return 47 })
+	if g := GetOrRegisterGaugeFloat64("foo", r); 47 != g.Value() {
+		t.Fatal(g)
+	}
+}

metrics/gauge_test.go

@@ -0,0 +1,68 @@
+package metrics
+
+import (
+	"fmt"
+	"testing"
+)
+
+func BenchmarkGuage(b *testing.B) {
+	g := NewGauge()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		g.Update(int64(i))
+	}
+}
+
+func TestGauge(t *testing.T) {
+	g := NewGauge()
+	g.Update(int64(47))
+	if v := g.Value(); 47 != v {
+		t.Errorf("g.Value(): 47 != %v\n", v)
+	}
+}
+
+func TestGaugeSnapshot(t *testing.T) {
+	g := NewGauge()
+	g.Update(int64(47))
+	snapshot := g.Snapshot()
+	g.Update(int64(0))
+	if v := snapshot.Value(); 47 != v {
+		t.Errorf("g.Value(): 47 != %v\n", v)
+	}
+}
+
+func TestGetOrRegisterGauge(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredGauge("foo", r).Update(47)
+	if g := GetOrRegisterGauge("foo", r); 47 != g.Value() {
+		t.Fatal(g)
+	}
+}
+
+func TestFunctionalGauge(t *testing.T) {
+	var counter int64
+	fg := NewFunctionalGauge(func() int64 {
+		counter++
+		return counter
+	})
+	fg.Value()
+	fg.Value()
+	if counter != 2 {
+		t.Error("counter != 2")
+	}
+}
+
+func TestGetOrRegisterFunctionalGauge(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredFunctionalGauge("foo", r, func() int64 { return 47 })
+	if g := GetOrRegisterGauge("foo", r); 47 != g.Value() {
+		t.Fatal(g)
+	}
+}
+
+func ExampleGetOrRegisterGauge() {
+	m := "server.bytes_sent"
+	g := GetOrRegisterGauge(m, nil)
+	g.Update(47)
+	fmt.Println(g.Value()) // Output: 47
+}

metrics/graphite.go

@@ -39,7 +39,7 @@ func Graphite(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) {
 // but it takes a GraphiteConfig instead.
 func GraphiteWithConfig(c GraphiteConfig) {
 	log.Printf("WARNING: This go-metrics client has been DEPRECATED! It has been moved to https://github.com/cyberdelia/go-metrics-graphite and will be removed from rcrowley/go-metrics on August 12th 2015")
-	for _ = range time.Tick(c.FlushInterval) {
+	for range time.Tick(c.FlushInterval) {
 		if err := graphite(&c); nil != err {
 			log.Println(err)
 		}

metrics/graphite_test.go

@@ -0,0 +1,22 @@
+package metrics
+
+import (
+	"net"
+	"time"
+)
+
+func ExampleGraphite() {
+	addr, _ := net.ResolveTCPAddr("net", ":2003")
+	go Graphite(DefaultRegistry, 1*time.Second, "some.prefix", addr)
+}
+
+func ExampleGraphiteWithConfig() {
+	addr, _ := net.ResolveTCPAddr("net", ":2003")
+	go GraphiteWithConfig(GraphiteConfig{
+		Addr:          addr,
+		Registry:      DefaultRegistry,
+		FlushInterval: 1 * time.Second,
+		DurationUnit:  time.Millisecond,
+		Percentiles:   []float64{0.5, 0.75, 0.99, 0.999},
+	})
+}

metrics/healthcheck.go

@@ -11,7 +11,7 @@ type Healthcheck interface {
 // NewHealthcheck constructs a new Healthcheck which will use the given
 // function to update its status.
 func NewHealthcheck(f func(Healthcheck)) Healthcheck {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilHealthcheck{}
 	}
 	return &StandardHealthcheck{nil, f}

metrics/histogram.go

@@ -28,7 +28,7 @@ func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram {
 
 // NewHistogram constructs a new StandardHistogram from a Sample.
 func NewHistogram(s Sample) Histogram {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilHistogram{}
 	}
 	return &StandardHistogram{sample: s}

metrics/histogram_test.go

@@ -0,0 +1,95 @@
+package metrics
+
+import "testing"
+
+func BenchmarkHistogram(b *testing.B) {
+	h := NewHistogram(NewUniformSample(100))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		h.Update(int64(i))
+	}
+}
+
+func TestGetOrRegisterHistogram(t *testing.T) {
+	r := NewRegistry()
+	s := NewUniformSample(100)
+	NewRegisteredHistogram("foo", r, s).Update(47)
+	if h := GetOrRegisterHistogram("foo", r, s); 1 != h.Count() {
+		t.Fatal(h)
+	}
+}
+
+func TestHistogram10000(t *testing.T) {
+	h := NewHistogram(NewUniformSample(100000))
+	for i := 1; i <= 10000; i++ {
+		h.Update(int64(i))
+	}
+	testHistogram10000(t, h)
+}
+
+func TestHistogramEmpty(t *testing.T) {
+	h := NewHistogram(NewUniformSample(100))
+	if count := h.Count(); 0 != count {
+		t.Errorf("h.Count(): 0 != %v\n", count)
+	}
+	if min := h.Min(); 0 != min {
+		t.Errorf("h.Min(): 0 != %v\n", min)
+	}
+	if max := h.Max(); 0 != max {
+		t.Errorf("h.Max(): 0 != %v\n", max)
+	}
+	if mean := h.Mean(); 0.0 != mean {
+		t.Errorf("h.Mean(): 0.0 != %v\n", mean)
+	}
+	if stdDev := h.StdDev(); 0.0 != stdDev {
+		t.Errorf("h.StdDev(): 0.0 != %v\n", stdDev)
+	}
+	ps := h.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 0.0 != ps[0] {
+		t.Errorf("median: 0.0 != %v\n", ps[0])
+	}
+	if 0.0 != ps[1] {
+		t.Errorf("75th percentile: 0.0 != %v\n", ps[1])
+	}
+	if 0.0 != ps[2] {
+		t.Errorf("99th percentile: 0.0 != %v\n", ps[2])
+	}
+}
+
+func TestHistogramSnapshot(t *testing.T) {
+	h := NewHistogram(NewUniformSample(100000))
+	for i := 1; i <= 10000; i++ {
+		h.Update(int64(i))
+	}
+	snapshot := h.Snapshot()
+	h.Update(0)
+	testHistogram10000(t, snapshot)
+}
+
+func testHistogram10000(t *testing.T, h Histogram) {
+	if count := h.Count(); 10000 != count {
+		t.Errorf("h.Count(): 10000 != %v\n", count)
+	}
+	if min := h.Min(); 1 != min {
+		t.Errorf("h.Min(): 1 != %v\n", min)
+	}
+	if max := h.Max(); 10000 != max {
+		t.Errorf("h.Max(): 10000 != %v\n", max)
+	}
+	if mean := h.Mean(); 5000.5 != mean {
+		t.Errorf("h.Mean(): 5000.5 != %v\n", mean)
+	}
+	if stdDev := h.StdDev(); 2886.751331514372 != stdDev {
+		t.Errorf("h.StdDev(): 2886.751331514372 != %v\n", stdDev)
+	}
+	ps := h.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 5000.5 != ps[0] {
+		t.Errorf("median: 5000.5 != %v\n", ps[0])
+	}
+	if 7500.75 != ps[1] {
+		t.Errorf("75th percentile: 7500.75 != %v\n", ps[1])
+	}
+	if 9900.99 != ps[2] {
+		t.Errorf("99th percentile: 9900.99 != %v\n", ps[2])
+	}
+}

metrics/influxdb/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2015 Vincent Rischmann
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

metrics/influxdb/README.md

@@ -0,0 +1,30 @@
+go-metrics-influxdb
+===================
+
+This is a reporter for the [go-metrics](https://github.com/rcrowley/go-metrics) library which will post the metrics to [InfluxDB](https://influxdb.com/).
+
+Note
+----
+
+This is only compatible with InfluxDB 0.9+.
+
+Usage
+-----
+
+```go
+import "github.com/vrischmann/go-metrics-influxdb"
+
+go influxdb.InfluxDB(
+    metrics.DefaultRegistry, // metrics registry
+    time.Second * 10,        // interval
+    "http://localhost:8086", // the InfluxDB url
+    "mydb",                  // your InfluxDB database
+    "myuser",                // your InfluxDB user
+    "mypassword",            // your InfluxDB password
+)
+```
+
+License
+-------
+
+go-metrics-influxdb is licensed under the MIT license. See the LICENSE file for details.

metrics/influxdb/influxdb.go

@@ -0,0 +1,227 @@
+package influxdb
+
+import (
+	"fmt"
+	"log"
+	uurl "net/url"
+	"time"
+
+	"github.com/EthereumCommonwealth/go-callisto/metrics"
+	"github.com/influxdata/influxdb/client"
+)
+
+type reporter struct {
+	reg      metrics.Registry
+	interval time.Duration
+
+	url       uurl.URL
+	database  string
+	username  string
+	password  string
+	namespace string
+	tags      map[string]string
+
+	client *client.Client
+
+	cache map[string]int64
+}
+
+// InfluxDB starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval.
+func InfluxDB(r metrics.Registry, d time.Duration, url, database, username, password, namespace string) {
+	InfluxDBWithTags(r, d, url, database, username, password, namespace, nil)
+}
+
+// InfluxDBWithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags
+func InfluxDBWithTags(r metrics.Registry, d time.Duration, url, database, username, password, namespace string, tags map[string]string) {
+	u, err := uurl.Parse(url)
+	if err != nil {
+		log.Printf("unable to parse InfluxDB url %s. err=%v", url, err)
+		return
+	}
+
+	rep := &reporter{
+		reg:       r,
+		interval:  d,
+		url:       *u,
+		database:  database,
+		username:  username,
+		password:  password,
+		namespace: namespace,
+		tags:      tags,
+		cache:     make(map[string]int64),
+	}
+	if err := rep.makeClient(); err != nil {
+		log.Printf("unable to make InfluxDB client. err=%v", err)
+		return
+	}
+
+	rep.run()
+}
+
+func (r *reporter) makeClient() (err error) {
+	r.client, err = client.NewClient(client.Config{
+		URL:      r.url,
+		Username: r.username,
+		Password: r.password,
+	})
+
+	return
+}
+
+func (r *reporter) run() {
+	intervalTicker := time.Tick(r.interval)
+	pingTicker := time.Tick(time.Second * 5)
+
+	for {
+		select {
+		case <-intervalTicker:
+			if err := r.send(); err != nil {
+				log.Printf("unable to send to InfluxDB. err=%v", err)
+			}
+		case <-pingTicker:
+			_, _, err := r.client.Ping()
+			if err != nil {
+				log.Printf("got error while sending a ping to InfluxDB, trying to recreate client. err=%v", err)
+
+				if err = r.makeClient(); err != nil {
+					log.Printf("unable to make InfluxDB client. err=%v", err)
+				}
+			}
+		}
+	}
+}
+
+func (r *reporter) send() error {
+	var pts []client.Point
+
+	r.reg.Each(func(name string, i interface{}) {
+		now := time.Now()
+		namespace := r.namespace
+
+		switch metric := i.(type) {
+		case metrics.Counter:
+			v := metric.Count()
+			l := r.cache[name]
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.count", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"value": v - l,
+				},
+				Time: now,
+			})
+			r.cache[name] = v
+		case metrics.Gauge:
+			ms := metric.Snapshot()
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.gauge", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"value": ms.Value(),
+				},
+				Time: now,
+			})
+		case metrics.GaugeFloat64:
+			ms := metric.Snapshot()
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.gauge", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"value": ms.Value(),
+				},
+				Time: now,
+			})
+		case metrics.Histogram:
+			ms := metric.Snapshot()
+			ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999})
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.histogram", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"count":    ms.Count(),
+					"max":      ms.Max(),
+					"mean":     ms.Mean(),
+					"min":      ms.Min(),
+					"stddev":   ms.StdDev(),
+					"variance": ms.Variance(),
+					"p50":      ps[0],
+					"p75":      ps[1],
+					"p95":      ps[2],
+					"p99":      ps[3],
+					"p999":     ps[4],
+					"p9999":    ps[5],
+				},
+				Time: now,
+			})
+		case metrics.Meter:
+			ms := metric.Snapshot()
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.meter", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"count": ms.Count(),
+					"m1":    ms.Rate1(),
+					"m5":    ms.Rate5(),
+					"m15":   ms.Rate15(),
+					"mean":  ms.RateMean(),
+				},
+				Time: now,
+			})
+		case metrics.Timer:
+			ms := metric.Snapshot()
+			ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999})
+			pts = append(pts, client.Point{
+				Measurement: fmt.Sprintf("%s%s.timer", namespace, name),
+				Tags:        r.tags,
+				Fields: map[string]interface{}{
+					"count":    ms.Count(),
+					"max":      ms.Max(),
+					"mean":     ms.Mean(),
+					"min":      ms.Min(),
+					"stddev":   ms.StdDev(),
+					"variance": ms.Variance(),
+					"p50":      ps[0],
+					"p75":      ps[1],
+					"p95":      ps[2],
+					"p99":      ps[3],
+					"p999":     ps[4],
+					"p9999":    ps[5],
+					"m1":       ms.Rate1(),
+					"m5":       ms.Rate5(),
+					"m15":      ms.Rate15(),
+					"meanrate": ms.RateMean(),
+				},
+				Time: now,
+			})
+		case metrics.ResettingTimer:
+			t := metric.Snapshot()
+
+			if len(t.Values()) > 0 {
+				ps := t.Percentiles([]float64{50, 95, 99})
+				val := t.Values()
+				pts = append(pts, client.Point{
+					Measurement: fmt.Sprintf("%s%s.span", namespace, name),
+					Tags:        r.tags,
+					Fields: map[string]interface{}{
+						"count": len(val),
+						"max":   val[len(val)-1],
+						"mean":  t.Mean(),
+						"min":   val[0],
+						"p50":   ps[0],
+						"p95":   ps[1],
+						"p99":   ps[2],
+					},
+					Time: now,
+				})
+			}
+		}
+	})
+
+	bps := client.BatchPoints{
+		Points:   pts,
+		Database: r.database,
+	}
+
+	_, err := r.client.Write(bps)
+	return err
+}

metrics/init_test.go

@@ -0,0 +1,5 @@
+package metrics
+
+func init() {
+	Enabled = true
+}

metrics/json.go

@@ -0,0 +1,31 @@
+package metrics
+
+import (
+	"encoding/json"
+	"io"
+	"time"
+)
+
+// MarshalJSON returns a byte slice containing a JSON representation of all
+// the metrics in the Registry.
+func (r *StandardRegistry) MarshalJSON() ([]byte, error) {
+	return json.Marshal(r.GetAll())
+}
+
+// WriteJSON writes metrics from the given registry  periodically to the
+// specified io.Writer as JSON.
+func WriteJSON(r Registry, d time.Duration, w io.Writer) {
+	for range time.Tick(d) {
+		WriteJSONOnce(r, w)
+	}
+}
+
+// WriteJSONOnce writes metrics from the given registry to the specified
+// io.Writer as JSON.
+func WriteJSONOnce(r Registry, w io.Writer) {
+	json.NewEncoder(w).Encode(r)
+}
+
+func (p *PrefixedRegistry) MarshalJSON() ([]byte, error) {
+	return json.Marshal(p.GetAll())
+}

metrics/json_test.go

@@ -0,0 +1,28 @@
+package metrics
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+)
+
+func TestRegistryMarshallJSON(t *testing.T) {
+	b := &bytes.Buffer{}
+	enc := json.NewEncoder(b)
+	r := NewRegistry()
+	r.Register("counter", NewCounter())
+	enc.Encode(r)
+	if s := b.String(); "{\"counter\":{\"count\":0}}\n" != s {
+		t.Fatalf(s)
+	}
+}
+
+func TestRegistryWriteJSONOnce(t *testing.T) {
+	r := NewRegistry()
+	r.Register("counter", NewCounter())
+	b := &bytes.Buffer{}
+	WriteJSONOnce(r, b)
+	if s := b.String(); s != "{\"counter\":{\"count\":0}}\n" {
+		t.Fail()
+	}
+}

metrics/librato/client.go

@@ -0,0 +1,102 @@
+package librato
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+const Operations = "operations"
+const OperationsShort = "ops"
+
+type LibratoClient struct {
+	Email, Token string
+}
+
+// property strings
+const (
+	// display attributes
+	Color             = "color"
+	DisplayMax        = "display_max"
+	DisplayMin        = "display_min"
+	DisplayUnitsLong  = "display_units_long"
+	DisplayUnitsShort = "display_units_short"
+	DisplayStacked    = "display_stacked"
+	DisplayTransform  = "display_transform"
+	// special gauge display attributes
+	SummarizeFunction = "summarize_function"
+	Aggregate         = "aggregate"
+
+	// metric keys
+	Name        = "name"
+	Period      = "period"
+	Description = "description"
+	DisplayName = "display_name"
+	Attributes  = "attributes"
+
+	// measurement keys
+	MeasureTime = "measure_time"
+	Source      = "source"
+	Value       = "value"
+
+	// special gauge keys
+	Count      = "count"
+	Sum        = "sum"
+	Max        = "max"
+	Min        = "min"
+	SumSquares = "sum_squares"
+
+	// batch keys
+	Counters = "counters"
+	Gauges   = "gauges"
+
+	MetricsPostUrl = "https://metrics-api.librato.com/v1/metrics"
+)
+
+type Measurement map[string]interface{}
+type Metric map[string]interface{}
+
+type Batch struct {
+	Gauges      []Measurement `json:"gauges,omitempty"`
+	Counters    []Measurement `json:"counters,omitempty"`
+	MeasureTime int64         `json:"measure_time"`
+	Source      string        `json:"source"`
+}
+
+func (self *LibratoClient) PostMetrics(batch Batch) (err error) {
+	var (
+		js   []byte
+		req  *http.Request
+		resp *http.Response
+	)
+
+	if len(batch.Counters) == 0 && len(batch.Gauges) == 0 {
+		return nil
+	}
+
+	if js, err = json.Marshal(batch); err != nil {
+		return
+	}
+
+	if req, err = http.NewRequest("POST", MetricsPostUrl, bytes.NewBuffer(js)); err != nil {
+		return
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.SetBasicAuth(self.Email, self.Token)
+
+	if resp, err = http.DefaultClient.Do(req); err != nil {
+		return
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		var body []byte
+		if body, err = ioutil.ReadAll(resp.Body); err != nil {
+			body = []byte(fmt.Sprintf("(could not fetch response body for error: %s)", err))
+		}
+		err = fmt.Errorf("Unable to post to Librato: %d %s %s", resp.StatusCode, resp.Status, string(body))
+	}
+	return
+}

metrics/librato/librato.go

@@ -0,0 +1,235 @@
+package librato
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"regexp"
+	"time"
+
+	"github.com/EthereumCommonwealth/go-callisto/metrics"
+)
+
+// a regexp for extracting the unit from time.Duration.String
+var unitRegexp = regexp.MustCompile(`[^\\d]+$`)
+
+// a helper that turns a time.Duration into librato display attributes for timer metrics
+func translateTimerAttributes(d time.Duration) (attrs map[string]interface{}) {
+	attrs = make(map[string]interface{})
+	attrs[DisplayTransform] = fmt.Sprintf("x/%d", int64(d))
+	attrs[DisplayUnitsShort] = string(unitRegexp.Find([]byte(d.String())))
+	return
+}
+
+type Reporter struct {
+	Email, Token    string
+	Namespace       string
+	Source          string
+	Interval        time.Duration
+	Registry        metrics.Registry
+	Percentiles     []float64              // percentiles to report on histogram metrics
+	TimerAttributes map[string]interface{} // units in which timers will be displayed
+	intervalSec     int64
+}
+
+func NewReporter(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) *Reporter {
+	return &Reporter{e, t, "", s, d, r, p, translateTimerAttributes(u), int64(d / time.Second)}
+}
+
+func Librato(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) {
+	NewReporter(r, d, e, t, s, p, u).Run()
+}
+
+func (self *Reporter) Run() {
+	log.Printf("WARNING: This client has been DEPRECATED! It has been moved to https://github.com/mihasya/go-metrics-librato and will be removed from rcrowley/go-metrics on August 5th 2015")
+	ticker := time.Tick(self.Interval)
+	metricsApi := &LibratoClient{self.Email, self.Token}
+	for now := range ticker {
+		var metrics Batch
+		var err error
+		if metrics, err = self.BuildRequest(now, self.Registry); err != nil {
+			log.Printf("ERROR constructing librato request body %s", err)
+			continue
+		}
+		if err := metricsApi.PostMetrics(metrics); err != nil {
+			log.Printf("ERROR sending metrics to librato %s", err)
+			continue
+		}
+	}
+}
+
+// calculate sum of squares from data provided by metrics.Histogram
+// see http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods
+func sumSquares(s metrics.Sample) float64 {
+	count := float64(s.Count())
+	sumSquared := math.Pow(count*s.Mean(), 2)
+	sumSquares := math.Pow(count*s.StdDev(), 2) + sumSquared/count
+	if math.IsNaN(sumSquares) {
+		return 0.0
+	}
+	return sumSquares
+}
+func sumSquaresTimer(t metrics.Timer) float64 {
+	count := float64(t.Count())
+	sumSquared := math.Pow(count*t.Mean(), 2)
+	sumSquares := math.Pow(count*t.StdDev(), 2) + sumSquared/count
+	if math.IsNaN(sumSquares) {
+		return 0.0
+	}
+	return sumSquares
+}
+
+func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Batch, err error) {
+	snapshot = Batch{
+		// coerce timestamps to a stepping fn so that they line up in Librato graphs
+		MeasureTime: (now.Unix() / self.intervalSec) * self.intervalSec,
+		Source:      self.Source,
+	}
+	snapshot.Gauges = make([]Measurement, 0)
+	snapshot.Counters = make([]Measurement, 0)
+	histogramGaugeCount := 1 + len(self.Percentiles)
+	r.Each(func(name string, metric interface{}) {
+		if self.Namespace != "" {
+			name = fmt.Sprintf("%s.%s", self.Namespace, name)
+		}
+		measurement := Measurement{}
+		measurement[Period] = self.Interval.Seconds()
+		switch m := metric.(type) {
+		case metrics.Counter:
+			if m.Count() > 0 {
+				measurement[Name] = fmt.Sprintf("%s.%s", name, "count")
+				measurement[Value] = float64(m.Count())
+				measurement[Attributes] = map[string]interface{}{
+					DisplayUnitsLong:  Operations,
+					DisplayUnitsShort: OperationsShort,
+					DisplayMin:        "0",
+				}
+				snapshot.Counters = append(snapshot.Counters, measurement)
+			}
+		case metrics.Gauge:
+			measurement[Name] = name
+			measurement[Value] = float64(m.Value())
+			snapshot.Gauges = append(snapshot.Gauges, measurement)
+		case metrics.GaugeFloat64:
+			measurement[Name] = name
+			measurement[Value] = m.Value()
+			snapshot.Gauges = append(snapshot.Gauges, measurement)
+		case metrics.Histogram:
+			if m.Count() > 0 {
+				gauges := make([]Measurement, histogramGaugeCount)
+				s := m.Sample()
+				measurement[Name] = fmt.Sprintf("%s.%s", name, "hist")
+				measurement[Count] = uint64(s.Count())
+				measurement[Max] = float64(s.Max())
+				measurement[Min] = float64(s.Min())
+				measurement[Sum] = float64(s.Sum())
+				measurement[SumSquares] = sumSquares(s)
+				gauges[0] = measurement
+				for i, p := range self.Percentiles {
+					gauges[i+1] = Measurement{
+						Name:   fmt.Sprintf("%s.%.2f", measurement[Name], p),
+						Value:  s.Percentile(p),
+						Period: measurement[Period],
+					}
+				}
+				snapshot.Gauges = append(snapshot.Gauges, gauges...)
+			}
+		case metrics.Meter:
+			measurement[Name] = name
+			measurement[Value] = float64(m.Count())
+			snapshot.Counters = append(snapshot.Counters, measurement)
+			snapshot.Gauges = append(snapshot.Gauges,
+				Measurement{
+					Name:   fmt.Sprintf("%s.%s", name, "1min"),
+					Value:  m.Rate1(),
+					Period: int64(self.Interval.Seconds()),
+					Attributes: map[string]interface{}{
+						DisplayUnitsLong:  Operations,
+						DisplayUnitsShort: OperationsShort,
+						DisplayMin:        "0",
+					},
+				},
+				Measurement{
+					Name:   fmt.Sprintf("%s.%s", name, "5min"),
+					Value:  m.Rate5(),
+					Period: int64(self.Interval.Seconds()),
+					Attributes: map[string]interface{}{
+						DisplayUnitsLong:  Operations,
+						DisplayUnitsShort: OperationsShort,
+						DisplayMin:        "0",
+					},
+				},
+				Measurement{
+					Name:   fmt.Sprintf("%s.%s", name, "15min"),
+					Value:  m.Rate15(),
+					Period: int64(self.Interval.Seconds()),
+					Attributes: map[string]interface{}{
+						DisplayUnitsLong:  Operations,
+						DisplayUnitsShort: OperationsShort,
+						DisplayMin:        "0",
+					},
+				},
+			)
+		case metrics.Timer:
+			measurement[Name] = name
+			measurement[Value] = float64(m.Count())
+			snapshot.Counters = append(snapshot.Counters, measurement)
+			if m.Count() > 0 {
+				libratoName := fmt.Sprintf("%s.%s", name, "timer.mean")
+				gauges := make([]Measurement, histogramGaugeCount)
+				gauges[0] = Measurement{
+					Name:       libratoName,
+					Count:      uint64(m.Count()),
+					Sum:        m.Mean() * float64(m.Count()),
+					Max:        float64(m.Max()),
+					Min:        float64(m.Min()),
+					SumSquares: sumSquaresTimer(m),
+					Period:     int64(self.Interval.Seconds()),
+					Attributes: self.TimerAttributes,
+				}
+				for i, p := range self.Percentiles {
+					gauges[i+1] = Measurement{
+						Name:       fmt.Sprintf("%s.timer.%2.0f", name, p*100),
+						Value:      m.Percentile(p),
+						Period:     int64(self.Interval.Seconds()),
+						Attributes: self.TimerAttributes,
+					}
+				}
+				snapshot.Gauges = append(snapshot.Gauges, gauges...)
+				snapshot.Gauges = append(snapshot.Gauges,
+					Measurement{
+						Name:   fmt.Sprintf("%s.%s", name, "rate.1min"),
+						Value:  m.Rate1(),
+						Period: int64(self.Interval.Seconds()),
+						Attributes: map[string]interface{}{
+							DisplayUnitsLong:  Operations,
+							DisplayUnitsShort: OperationsShort,
+							DisplayMin:        "0",
+						},
+					},
+					Measurement{
+						Name:   fmt.Sprintf("%s.%s", name, "rate.5min"),
+						Value:  m.Rate5(),
+						Period: int64(self.Interval.Seconds()),
+						Attributes: map[string]interface{}{
+							DisplayUnitsLong:  Operations,
+							DisplayUnitsShort: OperationsShort,
+							DisplayMin:        "0",
+						},
+					},
+					Measurement{
+						Name:   fmt.Sprintf("%s.%s", name, "rate.15min"),
+						Value:  m.Rate15(),
+						Period: int64(self.Interval.Seconds()),
+						Attributes: map[string]interface{}{
+							DisplayUnitsLong:  Operations,
+							DisplayUnitsShort: OperationsShort,
+							DisplayMin:        "0",
+						},
+					},
+				)
+			}
+		}
+	})
+	return
+}

metrics/log.go

@@ -18,7 +18,7 @@ func LogScaled(r Registry, freq time.Duration, scale time.Duration, l Logger) {
 	du := float64(scale)
 	duSuffix := scale.String()[1:]
 
-	for _ = range time.Tick(freq) {
+	for range time.Tick(freq) {
 		r.Each(func(name string, i interface{}) {
 			switch metric := i.(type) {
 			case Counter:

metrics/meter.go

@@ -15,10 +15,13 @@ type Meter interface {
 	Rate15() float64
 	RateMean() float64
 	Snapshot() Meter
+	Stop()
 }
 
 // GetOrRegisterMeter returns an existing Meter or constructs and registers a
 // new StandardMeter.
+// Be sure to unregister the meter from the registry once it is of no use to
+// allow for garbage collection.
 func GetOrRegisterMeter(name string, r Registry) Meter {
 	if nil == r {
 		r = DefaultRegistry
@@ -27,14 +30,15 @@ func GetOrRegisterMeter(name string, r Registry) Meter {
 }
 
 // NewMeter constructs a new StandardMeter and launches a goroutine.
+// Be sure to call Stop() once the meter is of no use to allow for garbage collection.
 func NewMeter() Meter {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilMeter{}
 	}
 	m := newStandardMeter()
 	arbiter.Lock()
 	defer arbiter.Unlock()
-	arbiter.meters = append(arbiter.meters, m)
+	arbiter.meters[m] = struct{}{}
 	if !arbiter.started {
 		arbiter.started = true
 		go arbiter.tick()
@@ -44,6 +48,8 @@ func NewMeter() Meter {
 
 // NewMeter constructs and registers a new StandardMeter and launches a
 // goroutine.
+// Be sure to unregister the meter from the registry once it is of no use to
+// allow for garbage collection.
 func NewRegisteredMeter(name string, r Registry) Meter {
 	c := NewMeter()
 	if nil == r {
@@ -86,6 +92,9 @@ func (m *MeterSnapshot) RateMean() float64 { return m.rateMean }
 // Snapshot returns the snapshot.
 func (m *MeterSnapshot) Snapshot() Meter { return m }
 
+// Stop is a no-op.
+func (m *MeterSnapshot) Stop() {}
+
 // NilMeter is a no-op Meter.
 type NilMeter struct{}
 
@@ -110,12 +119,16 @@ func (NilMeter) RateMean() float64 { return 0.0 }
 // Snapshot is a no-op.
 func (NilMeter) Snapshot() Meter { return NilMeter{} }
 
+// Stop is a no-op.
+func (NilMeter) Stop() {}
+
 // StandardMeter is the standard implementation of a Meter.
 type StandardMeter struct {
 	lock        sync.RWMutex
 	snapshot    *MeterSnapshot
 	a1, a5, a15 EWMA
 	startTime   time.Time
+	stopped     bool
 }
 
 func newStandardMeter() *StandardMeter {
@@ -128,6 +141,19 @@ func newStandardMeter() *StandardMeter {
 	}
 }
 
+// Stop stops the meter, Mark() will be a no-op if you use it after being stopped.
+func (m *StandardMeter) Stop() {
+	m.lock.Lock()
+	stopped := m.stopped
+	m.stopped = true
+	m.lock.Unlock()
+	if !stopped {
+		arbiter.Lock()
+		delete(arbiter.meters, m)
+		arbiter.Unlock()
+	}
+}
+
 // Count returns the number of events recorded.
 func (m *StandardMeter) Count() int64 {
 	m.lock.RLock()
@@ -136,10 +162,13 @@ func (m *StandardMeter) Count() int64 {
 	return count
 }
 
-// Mark records the occurance of n events.
+// Mark records the occurrence of n events.
 func (m *StandardMeter) Mark(n int64) {
 	m.lock.Lock()
 	defer m.lock.Unlock()
+	if m.stopped {
+		return
+	}
 	m.snapshot.count += n
 	m.a1.Update(n)
 	m.a5.Update(n)
@@ -205,29 +234,28 @@ func (m *StandardMeter) tick() {
 	m.updateSnapshot()
 }
 
+// meterArbiter ticks meters every 5s from a single goroutine.
+// meters are references in a set for future stopping.
 type meterArbiter struct {
 	sync.RWMutex
 	started bool
-	meters  []*StandardMeter
+	meters  map[*StandardMeter]struct{}
 	ticker  *time.Ticker
 }
 
-var arbiter = meterArbiter{ticker: time.NewTicker(5e9)}
+var arbiter = meterArbiter{ticker: time.NewTicker(5e9), meters: make(map[*StandardMeter]struct{})}
 
 // Ticks meters on the scheduled interval
 func (ma *meterArbiter) tick() {
-	for {
+	for range ma.ticker.C {
-		select {
+		ma.tickMeters()
-		case <-ma.ticker.C:
-			ma.tickMeters()
-		}
 	}
 }
 
 func (ma *meterArbiter) tickMeters() {
 	ma.RLock()
 	defer ma.RUnlock()
-	for _, meter := range ma.meters {
+	for meter := range ma.meters {
 		meter.tick()
 	}
 }

metrics/meter_test.go

@@ -0,0 +1,73 @@
+package metrics
+
+import (
+	"testing"
+	"time"
+)
+
+func BenchmarkMeter(b *testing.B) {
+	m := NewMeter()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.Mark(1)
+	}
+}
+
+func TestGetOrRegisterMeter(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredMeter("foo", r).Mark(47)
+	if m := GetOrRegisterMeter("foo", r); 47 != m.Count() {
+		t.Fatal(m)
+	}
+}
+
+func TestMeterDecay(t *testing.T) {
+	ma := meterArbiter{
+		ticker: time.NewTicker(time.Millisecond),
+		meters: make(map[*StandardMeter]struct{}),
+	}
+	m := newStandardMeter()
+	ma.meters[m] = struct{}{}
+	go ma.tick()
+	m.Mark(1)
+	rateMean := m.RateMean()
+	time.Sleep(100 * time.Millisecond)
+	if m.RateMean() >= rateMean {
+		t.Error("m.RateMean() didn't decrease")
+	}
+}
+
+func TestMeterNonzero(t *testing.T) {
+	m := NewMeter()
+	m.Mark(3)
+	if count := m.Count(); 3 != count {
+		t.Errorf("m.Count(): 3 != %v\n", count)
+	}
+}
+
+func TestMeterStop(t *testing.T) {
+	l := len(arbiter.meters)
+	m := NewMeter()
+	if len(arbiter.meters) != l+1 {
+		t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters))
+	}
+	m.Stop()
+	if len(arbiter.meters) != l {
+		t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters))
+	}
+}
+
+func TestMeterSnapshot(t *testing.T) {
+	m := NewMeter()
+	m.Mark(1)
+	if snapshot := m.Snapshot(); m.RateMean() != snapshot.RateMean() {
+		t.Fatal(snapshot)
+	}
+}
+
+func TestMeterZero(t *testing.T) {
+	m := NewMeter()
+	if count := m.Count(); 0 != count {
+		t.Errorf("m.Count(): 0 != %v\n", count)
+	}
+}

metrics/metrics.go

@@ -1,20 +1,8 @@
-// Copyright 2015 The go-ethereum Authors
+// Go port of Coda Hale's Metrics library
-// This file is part of the go-ethereum library.
 //
-// The go-ethereum library is free software: you can redistribute it and/or modify
+// <https://github.com/rcrowley/go-metrics>
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
 //
-// The go-ethereum library is distributed in the hope that it will be useful,
+// Coda Hale's original work: <https://github.com/codahale/metrics>
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package metrics provides general system and process level metrics collection.
 package metrics
 
 import (
@@ -24,17 +12,19 @@ import (
 	"time"
 
 	"github.com/EthereumCommonwealth/go-callisto/log"
-	"github.com/rcrowley/go-metrics"
-	"github.com/rcrowley/go-metrics/exp"
 )
 
+// Enabled is checked by the constructor functions for all of the
+// standard metrics.  If it is true, the metric returned is a stub.
+//
+// This global kill-switch helps quantify the observer effect and makes
+// for less cluttered pprof profiles.
+var Enabled bool = false
+
 // MetricsEnabledFlag is the CLI flag name to use to enable metrics collections.
 const MetricsEnabledFlag = "metrics"
 const DashboardEnabledFlag = "dashboard"
 
-// Enabled is the flag specifying if metrics are enable or not.
-var Enabled = false
-
 // Init enables or disables the metrics system. Since we need this to run before
 // any other code gets to create meters and timers, we'll actually do an ugly hack
 // and peek into the command line args for the metrics flag.
@@ -45,34 +35,6 @@ func init() {
 			Enabled = true
 		}
 	}
-	exp.Exp(metrics.DefaultRegistry)
-}
-
-// NewCounter create a new metrics Counter, either a real one of a NOP stub depending
-// on the metrics flag.
-func NewCounter(name string) metrics.Counter {
-	if !Enabled {
-		return new(metrics.NilCounter)
-	}
-	return metrics.GetOrRegisterCounter(name, metrics.DefaultRegistry)
-}
-
-// NewMeter create a new metrics Meter, either a real one of a NOP stub depending
-// on the metrics flag.
-func NewMeter(name string) metrics.Meter {
-	if !Enabled {
-		return new(metrics.NilMeter)
-	}
-	return metrics.GetOrRegisterMeter(name, metrics.DefaultRegistry)
-}
-
-// NewTimer create a new metrics Timer, either a real one of a NOP stub depending
-// on the metrics flag.
-func NewTimer(name string) metrics.Timer {
-	if !Enabled {
-		return new(metrics.NilTimer)
-	}
-	return metrics.GetOrRegisterTimer(name, metrics.DefaultRegistry)
 }
 
 // CollectProcessMetrics periodically collects various metrics about the running
@@ -90,17 +52,17 @@ func CollectProcessMetrics(refresh time.Duration) {
 		diskstats[i] = new(DiskStats)
 	}
 	// Define the various metrics to collect
-	memAllocs := metrics.GetOrRegisterMeter("system/memory/allocs", metrics.DefaultRegistry)
+	memAllocs := GetOrRegisterMeter("system/memory/allocs", DefaultRegistry)
-	memFrees := metrics.GetOrRegisterMeter("system/memory/frees", metrics.DefaultRegistry)
+	memFrees := GetOrRegisterMeter("system/memory/frees", DefaultRegistry)
-	memInuse := metrics.GetOrRegisterMeter("system/memory/inuse", metrics.DefaultRegistry)
+	memInuse := GetOrRegisterMeter("system/memory/inuse", DefaultRegistry)
-	memPauses := metrics.GetOrRegisterMeter("system/memory/pauses", metrics.DefaultRegistry)
+	memPauses := GetOrRegisterMeter("system/memory/pauses", DefaultRegistry)
 
-	var diskReads, diskReadBytes, diskWrites, diskWriteBytes metrics.Meter
+	var diskReads, diskReadBytes, diskWrites, diskWriteBytes Meter
 	if err := ReadDiskStats(diskstats[0]); err == nil {
-		diskReads = metrics.GetOrRegisterMeter("system/disk/readcount", metrics.DefaultRegistry)
+		diskReads = GetOrRegisterMeter("system/disk/readcount", DefaultRegistry)
-		diskReadBytes = metrics.GetOrRegisterMeter("system/disk/readdata", metrics.DefaultRegistry)
+		diskReadBytes = GetOrRegisterMeter("system/disk/readdata", DefaultRegistry)
-		diskWrites = metrics.GetOrRegisterMeter("system/disk/writecount", metrics.DefaultRegistry)
+		diskWrites = GetOrRegisterMeter("system/disk/writecount", DefaultRegistry)
-		diskWriteBytes = metrics.GetOrRegisterMeter("system/disk/writedata", metrics.DefaultRegistry)
+		diskWriteBytes = GetOrRegisterMeter("system/disk/writedata", DefaultRegistry)
 	} else {
 		log.Debug("Failed to read disk metrics", "err", err)
 	}

metrics/metrics_test.go

@@ -0,0 +1,124 @@
+package metrics
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"sync"
+	"testing"
+)
+
+const FANOUT = 128
+
+// Stop the compiler from complaining during debugging.
+var (
+	_ = ioutil.Discard
+	_ = log.LstdFlags
+)
+
+func BenchmarkMetrics(b *testing.B) {
+	r := NewRegistry()
+	c := NewRegisteredCounter("counter", r)
+	g := NewRegisteredGauge("gauge", r)
+	gf := NewRegisteredGaugeFloat64("gaugefloat64", r)
+	h := NewRegisteredHistogram("histogram", r, NewUniformSample(100))
+	m := NewRegisteredMeter("meter", r)
+	t := NewRegisteredTimer("timer", r)
+	RegisterDebugGCStats(r)
+	RegisterRuntimeMemStats(r)
+	b.ResetTimer()
+	ch := make(chan bool)
+
+	wgD := &sync.WaitGroup{}
+	/*
+		wgD.Add(1)
+		go func() {
+			defer wgD.Done()
+			//log.Println("go CaptureDebugGCStats")
+			for {
+				select {
+				case <-ch:
+					//log.Println("done CaptureDebugGCStats")
+					return
+				default:
+					CaptureDebugGCStatsOnce(r)
+				}
+			}
+		}()
+	//*/
+
+	wgR := &sync.WaitGroup{}
+	//*
+	wgR.Add(1)
+	go func() {
+		defer wgR.Done()
+		//log.Println("go CaptureRuntimeMemStats")
+		for {
+			select {
+			case <-ch:
+				//log.Println("done CaptureRuntimeMemStats")
+				return
+			default:
+				CaptureRuntimeMemStatsOnce(r)
+			}
+		}
+	}()
+	//*/
+
+	wgW := &sync.WaitGroup{}
+	/*
+		wgW.Add(1)
+		go func() {
+			defer wgW.Done()
+			//log.Println("go Write")
+			for {
+				select {
+				case <-ch:
+					//log.Println("done Write")
+					return
+				default:
+					WriteOnce(r, ioutil.Discard)
+				}
+			}
+		}()
+	//*/
+
+	wg := &sync.WaitGroup{}
+	wg.Add(FANOUT)
+	for i := 0; i < FANOUT; i++ {
+		go func(i int) {
+			defer wg.Done()
+			//log.Println("go", i)
+			for i := 0; i < b.N; i++ {
+				c.Inc(1)
+				g.Update(int64(i))
+				gf.Update(float64(i))
+				h.Update(int64(i))
+				m.Mark(1)
+				t.Update(1)
+			}
+			//log.Println("done", i)
+		}(i)
+	}
+	wg.Wait()
+	close(ch)
+	wgD.Wait()
+	wgR.Wait()
+	wgW.Wait()
+}
+
+func Example() {
+	c := NewCounter()
+	Register("money", c)
+	c.Inc(17)
+
+	// Threadsafe registration
+	t := GetOrRegisterTimer("db.get.latency", nil)
+	t.Time(func() {})
+	t.Update(1)
+
+	fmt.Println(c.Count())
+	fmt.Println(t.Min())
+	// Output: 17
+	// 1
+}

metrics/opentsdb.go

@@ -38,7 +38,7 @@ func OpenTSDB(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) {
 // OpenTSDBWithConfig is a blocking exporter function just like OpenTSDB,
 // but it takes a OpenTSDBConfig instead.
 func OpenTSDBWithConfig(c OpenTSDBConfig) {
-	for _ = range time.Tick(c.FlushInterval) {
+	for range time.Tick(c.FlushInterval) {
 		if err := openTSDB(&c); nil != err {
 			log.Println(err)
 		}

metrics/opentsdb_test.go

@@ -0,0 +1,21 @@
+package metrics
+
+import (
+	"net"
+	"time"
+)
+
+func ExampleOpenTSDB() {
+	addr, _ := net.ResolveTCPAddr("net", ":2003")
+	go OpenTSDB(DefaultRegistry, 1*time.Second, "some.prefix", addr)
+}
+
+func ExampleOpenTSDBWithConfig() {
+	addr, _ := net.ResolveTCPAddr("net", ":2003")
+	go OpenTSDBWithConfig(OpenTSDBConfig{
+		Addr:          addr,
+		Registry:      DefaultRegistry,
+		FlushInterval: 1 * time.Second,
+		DurationUnit:  time.Millisecond,
+	})
+}

metrics/registry.go

@@ -29,6 +29,9 @@ type Registry interface {
 	// Get the metric by the given name or nil if none is registered.
 	Get(string) interface{}
 
+	// GetAll metrics in the Registry.
+	GetAll() map[string]map[string]interface{}
+
 	// Gets an existing metric or registers the given one.
 	// The interface can be the metric to register if not found in registry,
 	// or a function returning the metric for lazy instantiation.
@@ -109,10 +112,72 @@ func (r *StandardRegistry) RunHealthchecks() {
 	}
 }
 
+// GetAll metrics in the Registry
+func (r *StandardRegistry) GetAll() map[string]map[string]interface{} {
+	data := make(map[string]map[string]interface{})
+	r.Each(func(name string, i interface{}) {
+		values := make(map[string]interface{})
+		switch metric := i.(type) {
+		case Counter:
+			values["count"] = metric.Count()
+		case Gauge:
+			values["value"] = metric.Value()
+		case GaugeFloat64:
+			values["value"] = metric.Value()
+		case Healthcheck:
+			values["error"] = nil
+			metric.Check()
+			if err := metric.Error(); nil != err {
+				values["error"] = metric.Error().Error()
+			}
+		case Histogram:
+			h := metric.Snapshot()
+			ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			values["count"] = h.Count()
+			values["min"] = h.Min()
+			values["max"] = h.Max()
+			values["mean"] = h.Mean()
+			values["stddev"] = h.StdDev()
+			values["median"] = ps[0]
+			values["75%"] = ps[1]
+			values["95%"] = ps[2]
+			values["99%"] = ps[3]
+			values["99.9%"] = ps[4]
+		case Meter:
+			m := metric.Snapshot()
+			values["count"] = m.Count()
+			values["1m.rate"] = m.Rate1()
+			values["5m.rate"] = m.Rate5()
+			values["15m.rate"] = m.Rate15()
+			values["mean.rate"] = m.RateMean()
+		case Timer:
+			t := metric.Snapshot()
+			ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			values["count"] = t.Count()
+			values["min"] = t.Min()
+			values["max"] = t.Max()
+			values["mean"] = t.Mean()
+			values["stddev"] = t.StdDev()
+			values["median"] = ps[0]
+			values["75%"] = ps[1]
+			values["95%"] = ps[2]
+			values["99%"] = ps[3]
+			values["99.9%"] = ps[4]
+			values["1m.rate"] = t.Rate1()
+			values["5m.rate"] = t.Rate5()
+			values["15m.rate"] = t.Rate15()
+			values["mean.rate"] = t.RateMean()
+		}
+		data[name] = values
+	})
+	return data
+}
+
 // Unregister the metric with the given name.
 func (r *StandardRegistry) Unregister(name string) {
 	r.mutex.Lock()
 	defer r.mutex.Unlock()
+	r.stop(name)
 	delete(r.metrics, name)
 }
 
@@ -120,7 +185,8 @@ func (r *StandardRegistry) Unregister(name string) {
 func (r *StandardRegistry) UnregisterAll() {
 	r.mutex.Lock()
 	defer r.mutex.Unlock()
-	for name, _ := range r.metrics {
+	for name := range r.metrics {
+		r.stop(name)
 		delete(r.metrics, name)
 	}
 }
@@ -130,7 +196,7 @@ func (r *StandardRegistry) register(name string, i interface{}) error {
 		return DuplicateMetric(name)
 	}
 	switch i.(type) {
-	case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer:
+	case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer, ResettingTimer:
 		r.metrics[name] = i
 	}
 	return nil
@@ -146,6 +212,19 @@ func (r *StandardRegistry) registered() map[string]interface{} {
 	return metrics
 }
 
+func (r *StandardRegistry) stop(name string) {
+	if i, ok := r.metrics[name]; ok {
+		if s, ok := i.(Stoppable); ok {
+			s.Stop()
+		}
+	}
+}
+
+// Stoppable defines the metrics which has to be stopped.
+type Stoppable interface {
+	Stop()
+}
+
 type PrefixedRegistry struct {
 	underlying Registry
 	prefix     string
@@ -216,6 +295,11 @@ func (r *PrefixedRegistry) RunHealthchecks() {
 	r.underlying.RunHealthchecks()
 }
 
+// GetAll metrics in the Registry
+func (r *PrefixedRegistry) GetAll() map[string]map[string]interface{} {
+	return r.underlying.GetAll()
+}
+
 // Unregister the metric with the given name. The name will be prefixed.
 func (r *PrefixedRegistry) Unregister(name string) {
 	realName := r.prefix + name

metrics/registry_test.go

@@ -0,0 +1,305 @@
+package metrics
+
+import (
+	"testing"
+)
+
+func BenchmarkRegistry(b *testing.B) {
+	r := NewRegistry()
+	r.Register("foo", NewCounter())
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		r.Each(func(string, interface{}) {})
+	}
+}
+
+func TestRegistry(t *testing.T) {
+	r := NewRegistry()
+	r.Register("foo", NewCounter())
+	i := 0
+	r.Each(func(name string, iface interface{}) {
+		i++
+		if "foo" != name {
+			t.Fatal(name)
+		}
+		if _, ok := iface.(Counter); !ok {
+			t.Fatal(iface)
+		}
+	})
+	if 1 != i {
+		t.Fatal(i)
+	}
+	r.Unregister("foo")
+	i = 0
+	r.Each(func(string, interface{}) { i++ })
+	if 0 != i {
+		t.Fatal(i)
+	}
+}
+
+func TestRegistryDuplicate(t *testing.T) {
+	r := NewRegistry()
+	if err := r.Register("foo", NewCounter()); nil != err {
+		t.Fatal(err)
+	}
+	if err := r.Register("foo", NewGauge()); nil == err {
+		t.Fatal(err)
+	}
+	i := 0
+	r.Each(func(name string, iface interface{}) {
+		i++
+		if _, ok := iface.(Counter); !ok {
+			t.Fatal(iface)
+		}
+	})
+	if 1 != i {
+		t.Fatal(i)
+	}
+}
+
+func TestRegistryGet(t *testing.T) {
+	r := NewRegistry()
+	r.Register("foo", NewCounter())
+	if count := r.Get("foo").(Counter).Count(); 0 != count {
+		t.Fatal(count)
+	}
+	r.Get("foo").(Counter).Inc(1)
+	if count := r.Get("foo").(Counter).Count(); 1 != count {
+		t.Fatal(count)
+	}
+}
+
+func TestRegistryGetOrRegister(t *testing.T) {
+	r := NewRegistry()
+
+	// First metric wins with GetOrRegister
+	_ = r.GetOrRegister("foo", NewCounter())
+	m := r.GetOrRegister("foo", NewGauge())
+	if _, ok := m.(Counter); !ok {
+		t.Fatal(m)
+	}
+
+	i := 0
+	r.Each(func(name string, iface interface{}) {
+		i++
+		if name != "foo" {
+			t.Fatal(name)
+		}
+		if _, ok := iface.(Counter); !ok {
+			t.Fatal(iface)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestRegistryGetOrRegisterWithLazyInstantiation(t *testing.T) {
+	r := NewRegistry()
+
+	// First metric wins with GetOrRegister
+	_ = r.GetOrRegister("foo", NewCounter)
+	m := r.GetOrRegister("foo", NewGauge)
+	if _, ok := m.(Counter); !ok {
+		t.Fatal(m)
+	}
+
+	i := 0
+	r.Each(func(name string, iface interface{}) {
+		i++
+		if name != "foo" {
+			t.Fatal(name)
+		}
+		if _, ok := iface.(Counter); !ok {
+			t.Fatal(iface)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestRegistryUnregister(t *testing.T) {
+	l := len(arbiter.meters)
+	r := NewRegistry()
+	r.Register("foo", NewCounter())
+	r.Register("bar", NewMeter())
+	r.Register("baz", NewTimer())
+	if len(arbiter.meters) != l+2 {
+		t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters))
+	}
+	r.Unregister("foo")
+	r.Unregister("bar")
+	r.Unregister("baz")
+	if len(arbiter.meters) != l {
+		t.Errorf("arbiter.meters: %d != %d\n", l+2, len(arbiter.meters))
+	}
+}
+
+func TestPrefixedChildRegistryGetOrRegister(t *testing.T) {
+	r := NewRegistry()
+	pr := NewPrefixedChildRegistry(r, "prefix.")
+
+	_ = pr.GetOrRegister("foo", NewCounter())
+
+	i := 0
+	r.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.foo" {
+			t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestPrefixedRegistryGetOrRegister(t *testing.T) {
+	r := NewPrefixedRegistry("prefix.")
+
+	_ = r.GetOrRegister("foo", NewCounter())
+
+	i := 0
+	r.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.foo" {
+			t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestPrefixedRegistryRegister(t *testing.T) {
+	r := NewPrefixedRegistry("prefix.")
+	err := r.Register("foo", NewCounter())
+	c := NewCounter()
+	Register("bar", c)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	i := 0
+	r.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.foo" {
+			t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestPrefixedRegistryUnregister(t *testing.T) {
+	r := NewPrefixedRegistry("prefix.")
+
+	_ = r.Register("foo", NewCounter())
+
+	i := 0
+	r.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.foo" {
+			t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+
+	r.Unregister("foo")
+
+	i = 0
+	r.Each(func(name string, m interface{}) {
+		i++
+	})
+
+	if i != 0 {
+		t.Fatal(i)
+	}
+}
+
+func TestPrefixedRegistryGet(t *testing.T) {
+	pr := NewPrefixedRegistry("prefix.")
+	name := "foo"
+	pr.Register(name, NewCounter())
+
+	fooCounter := pr.Get(name)
+	if fooCounter == nil {
+		t.Fatal(name)
+	}
+}
+
+func TestPrefixedChildRegistryGet(t *testing.T) {
+	r := NewRegistry()
+	pr := NewPrefixedChildRegistry(r, "prefix.")
+	name := "foo"
+	pr.Register(name, NewCounter())
+	fooCounter := pr.Get(name)
+	if fooCounter == nil {
+		t.Fatal(name)
+	}
+}
+
+func TestChildPrefixedRegistryRegister(t *testing.T) {
+	r := NewPrefixedChildRegistry(DefaultRegistry, "prefix.")
+	err := r.Register("foo", NewCounter())
+	c := NewCounter()
+	Register("bar", c)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	i := 0
+	r.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.foo" {
+			t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestChildPrefixedRegistryOfChildRegister(t *testing.T) {
+	r := NewPrefixedChildRegistry(NewRegistry(), "prefix.")
+	r2 := NewPrefixedChildRegistry(r, "prefix2.")
+	err := r.Register("foo2", NewCounter())
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	err = r2.Register("baz", NewCounter())
+	c := NewCounter()
+	Register("bars", c)
+
+	i := 0
+	r2.Each(func(name string, m interface{}) {
+		i++
+		if name != "prefix.prefix2.baz" {
+			//t.Fatal(name)
+		}
+	})
+	if i != 1 {
+		t.Fatal(i)
+	}
+}
+
+func TestWalkRegistries(t *testing.T) {
+	r := NewPrefixedChildRegistry(NewRegistry(), "prefix.")
+	r2 := NewPrefixedChildRegistry(r, "prefix2.")
+	err := r.Register("foo2", NewCounter())
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	err = r2.Register("baz", NewCounter())
+	c := NewCounter()
+	Register("bars", c)
+
+	_, prefix := findPrefix(r2, "")
+	if "prefix.prefix2." != prefix {
+		t.Fatal(prefix)
+	}
+
+}

metrics/resetting_timer.go

@@ -0,0 +1,237 @@
+package metrics
+
+import (
+	"math"
+	"sort"
+	"sync"
+	"time"
+)
+
+// Initial slice capacity for the values stored in a ResettingTimer
+const InitialResettingTimerSliceCap = 10
+
+// ResettingTimer is used for storing aggregated values for timers, which are reset on every flush interval.
+type ResettingTimer interface {
+	Values() []int64
+	Snapshot() ResettingTimer
+	Percentiles([]float64) []int64
+	Mean() float64
+	Time(func())
+	Update(time.Duration)
+	UpdateSince(time.Time)
+}
+
+// GetOrRegisterResettingTimer returns an existing ResettingTimer or constructs and registers a
+// new StandardResettingTimer.
+func GetOrRegisterResettingTimer(name string, r Registry) ResettingTimer {
+	if nil == r {
+		r = DefaultRegistry
+	}
+	return r.GetOrRegister(name, NewResettingTimer).(ResettingTimer)
+}
+
+// NewRegisteredResettingTimer constructs and registers a new StandardResettingTimer.
+func NewRegisteredResettingTimer(name string, r Registry) ResettingTimer {
+	c := NewResettingTimer()
+	if nil == r {
+		r = DefaultRegistry
+	}
+	r.Register(name, c)
+	return c
+}
+
+// NewResettingTimer constructs a new StandardResettingTimer
+func NewResettingTimer() ResettingTimer {
+	if !Enabled {
+		return NilResettingTimer{}
+	}
+	return &StandardResettingTimer{
+		values: make([]int64, 0, InitialResettingTimerSliceCap),
+	}
+}
+
+// NilResettingTimer is a no-op ResettingTimer.
+type NilResettingTimer struct {
+}
+
+// Values is a no-op.
+func (NilResettingTimer) Values() []int64 { return nil }
+
+// Snapshot is a no-op.
+func (NilResettingTimer) Snapshot() ResettingTimer { return NilResettingTimer{} }
+
+// Time is a no-op.
+func (NilResettingTimer) Time(func()) {}
+
+// Update is a no-op.
+func (NilResettingTimer) Update(time.Duration) {}
+
+// Percentiles panics.
+func (NilResettingTimer) Percentiles([]float64) []int64 {
+	panic("Percentiles called on a NilResettingTimer")
+}
+
+// Mean panics.
+func (NilResettingTimer) Mean() float64 {
+	panic("Mean called on a NilResettingTimer")
+}
+
+// UpdateSince is a no-op.
+func (NilResettingTimer) UpdateSince(time.Time) {}
+
+// StandardResettingTimer is the standard implementation of a ResettingTimer.
+// and Meter.
+type StandardResettingTimer struct {
+	values []int64
+	mutex  sync.Mutex
+}
+
+// Values returns a slice with all measurements.
+func (t *StandardResettingTimer) Values() []int64 {
+	return t.values
+}
+
+// Snapshot resets the timer and returns a read-only copy of its contents.
+func (t *StandardResettingTimer) Snapshot() ResettingTimer {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	currentValues := t.values
+	t.values = make([]int64, 0, InitialResettingTimerSliceCap)
+
+	return &ResettingTimerSnapshot{
+		values: currentValues,
+	}
+}
+
+// Percentiles panics.
+func (t *StandardResettingTimer) Percentiles([]float64) []int64 {
+	panic("Percentiles called on a StandardResettingTimer")
+}
+
+// Mean panics.
+func (t *StandardResettingTimer) Mean() float64 {
+	panic("Mean called on a StandardResettingTimer")
+}
+
+// Record the duration of the execution of the given function.
+func (t *StandardResettingTimer) Time(f func()) {
+	ts := time.Now()
+	f()
+	t.Update(time.Since(ts))
+}
+
+// Record the duration of an event.
+func (t *StandardResettingTimer) Update(d time.Duration) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	t.values = append(t.values, int64(d))
+}
+
+// Record the duration of an event that started at a time and ends now.
+func (t *StandardResettingTimer) UpdateSince(ts time.Time) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	t.values = append(t.values, int64(time.Since(ts)))
+}
+
+// ResettingTimerSnapshot is a point-in-time copy of another ResettingTimer.
+type ResettingTimerSnapshot struct {
+	values              []int64
+	mean                float64
+	thresholdBoundaries []int64
+	calculated          bool
+}
+
+// Snapshot returns the snapshot.
+func (t *ResettingTimerSnapshot) Snapshot() ResettingTimer { return t }
+
+// Time panics.
+func (*ResettingTimerSnapshot) Time(func()) {
+	panic("Time called on a ResettingTimerSnapshot")
+}
+
+// Update panics.
+func (*ResettingTimerSnapshot) Update(time.Duration) {
+	panic("Update called on a ResettingTimerSnapshot")
+}
+
+// UpdateSince panics.
+func (*ResettingTimerSnapshot) UpdateSince(time.Time) {
+	panic("UpdateSince called on a ResettingTimerSnapshot")
+}
+
+// Values returns all values from snapshot.
+func (t *ResettingTimerSnapshot) Values() []int64 {
+	return t.values
+}
+
+// Percentiles returns the boundaries for the input percentiles.
+func (t *ResettingTimerSnapshot) Percentiles(percentiles []float64) []int64 {
+	t.calc(percentiles)
+
+	return t.thresholdBoundaries
+}
+
+// Mean returns the mean of the snapshotted values
+func (t *ResettingTimerSnapshot) Mean() float64 {
+	if !t.calculated {
+		t.calc([]float64{})
+	}
+
+	return t.mean
+}
+
+func (t *ResettingTimerSnapshot) calc(percentiles []float64) {
+	sort.Sort(Int64Slice(t.values))
+
+	count := len(t.values)
+	if count > 0 {
+		min := t.values[0]
+		max := t.values[count-1]
+
+		cumulativeValues := make([]int64, count)
+		cumulativeValues[0] = min
+		for i := 1; i < count; i++ {
+			cumulativeValues[i] = t.values[i] + cumulativeValues[i-1]
+		}
+
+		t.thresholdBoundaries = make([]int64, len(percentiles))
+
+		thresholdBoundary := max
+
+		for i, pct := range percentiles {
+			if count > 1 {
+				var abs float64
+				if pct >= 0 {
+					abs = pct
+				} else {
+					abs = 100 + pct
+				}
+				// poor man's math.Round(x):
+				// math.Floor(x + 0.5)
+				indexOfPerc := int(math.Floor(((abs / 100.0) * float64(count)) + 0.5))
+				if pct >= 0 {
+					indexOfPerc -= 1 // index offset=0
+				}
+				thresholdBoundary = t.values[indexOfPerc]
+			}
+
+			t.thresholdBoundaries[i] = thresholdBoundary
+		}
+
+		sum := cumulativeValues[count-1]
+		t.mean = float64(sum) / float64(count)
+	} else {
+		t.thresholdBoundaries = make([]int64, len(percentiles))
+		t.mean = 0
+	}
+
+	t.calculated = true
+}
+
+// Int64Slice attaches the methods of sort.Interface to []int64, sorting in increasing order.
+type Int64Slice []int64
+
+func (s Int64Slice) Len() int           { return len(s) }
+func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] }
+func (s Int64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

metrics/resetting_timer_test.go

@@ -0,0 +1,106 @@
+package metrics
+
+import (
+	"testing"
+	"time"
+)
+
+func TestResettingTimer(t *testing.T) {
+	tests := []struct {
+		values   []int64
+		start    int
+		end      int
+		wantP50  int64
+		wantP95  int64
+		wantP99  int64
+		wantMean float64
+		wantMin  int64
+		wantMax  int64
+	}{
+		{
+			values:  []int64{},
+			start:   1,
+			end:     11,
+			wantP50: 5, wantP95: 10, wantP99: 10,
+			wantMin: 1, wantMax: 10, wantMean: 5.5,
+		},
+		{
+			values:  []int64{},
+			start:   1,
+			end:     101,
+			wantP50: 50, wantP95: 95, wantP99: 99,
+			wantMin: 1, wantMax: 100, wantMean: 50.5,
+		},
+		{
+			values:  []int64{1},
+			start:   0,
+			end:     0,
+			wantP50: 1, wantP95: 1, wantP99: 1,
+			wantMin: 1, wantMax: 1, wantMean: 1,
+		},
+		{
+			values:  []int64{0},
+			start:   0,
+			end:     0,
+			wantP50: 0, wantP95: 0, wantP99: 0,
+			wantMin: 0, wantMax: 0, wantMean: 0,
+		},
+		{
+			values:  []int64{},
+			start:   0,
+			end:     0,
+			wantP50: 0, wantP95: 0, wantP99: 0,
+			wantMin: 0, wantMax: 0, wantMean: 0,
+		},
+		{
+			values:  []int64{1, 10},
+			start:   0,
+			end:     0,
+			wantP50: 1, wantP95: 10, wantP99: 10,
+			wantMin: 1, wantMax: 10, wantMean: 5.5,
+		},
+	}
+	for ind, tt := range tests {
+		timer := NewResettingTimer()
+
+		for i := tt.start; i < tt.end; i++ {
+			tt.values = append(tt.values, int64(i))
+		}
+
+		for _, v := range tt.values {
+			timer.Update(time.Duration(v))
+		}
+
+		snap := timer.Snapshot()
+
+		ps := snap.Percentiles([]float64{50, 95, 99})
+
+		val := snap.Values()
+
+		if len(val) > 0 {
+			if tt.wantMin != val[0] {
+				t.Fatalf("%d: min: got %d, want %d", ind, val[0], tt.wantMin)
+			}
+
+			if tt.wantMax != val[len(val)-1] {
+				t.Fatalf("%d: max: got %d, want %d", ind, val[len(val)-1], tt.wantMax)
+			}
+		}
+
+		if tt.wantMean != snap.Mean() {
+			t.Fatalf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean)
+		}
+
+		if tt.wantP50 != ps[0] {
+			t.Fatalf("%d: p50: got %d, want %d", ind, ps[0], tt.wantP50)
+		}
+
+		if tt.wantP95 != ps[1] {
+			t.Fatalf("%d: p95: got %d, want %d", ind, ps[1], tt.wantP95)
+		}
+
+		if tt.wantP99 != ps[2] {
+			t.Fatalf("%d: p99: got %d, want %d", ind, ps[2], tt.wantP99)
+		}
+	}
+}

metrics/runtime.go

@@ -55,7 +55,7 @@ var (
 // Capture new values for the Go runtime statistics exported in
 // runtime.MemStats.  This is designed to be called as a goroutine.
 func CaptureRuntimeMemStats(r Registry, d time.Duration) {
-	for _ = range time.Tick(d) {
+	for range time.Tick(d) {
 		CaptureRuntimeMemStatsOnce(r)
 	}
 }

metrics/runtime_test.go

@@ -0,0 +1,88 @@
+package metrics
+
+import (
+	"runtime"
+	"testing"
+	"time"
+)
+
+func BenchmarkRuntimeMemStats(b *testing.B) {
+	r := NewRegistry()
+	RegisterRuntimeMemStats(r)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		CaptureRuntimeMemStatsOnce(r)
+	}
+}
+
+func TestRuntimeMemStats(t *testing.T) {
+	r := NewRegistry()
+	RegisterRuntimeMemStats(r)
+	CaptureRuntimeMemStatsOnce(r)
+	zero := runtimeMetrics.MemStats.PauseNs.Count() // Get a "zero" since GC may have run before these tests.
+	runtime.GC()
+	CaptureRuntimeMemStatsOnce(r)
+	if count := runtimeMetrics.MemStats.PauseNs.Count(); 1 != count-zero {
+		t.Fatal(count - zero)
+	}
+	runtime.GC()
+	runtime.GC()
+	CaptureRuntimeMemStatsOnce(r)
+	if count := runtimeMetrics.MemStats.PauseNs.Count(); 3 != count-zero {
+		t.Fatal(count - zero)
+	}
+	for i := 0; i < 256; i++ {
+		runtime.GC()
+	}
+	CaptureRuntimeMemStatsOnce(r)
+	if count := runtimeMetrics.MemStats.PauseNs.Count(); 259 != count-zero {
+		t.Fatal(count - zero)
+	}
+	for i := 0; i < 257; i++ {
+		runtime.GC()
+	}
+	CaptureRuntimeMemStatsOnce(r)
+	if count := runtimeMetrics.MemStats.PauseNs.Count(); 515 != count-zero { // We lost one because there were too many GCs between captures.
+		t.Fatal(count - zero)
+	}
+}
+
+func TestRuntimeMemStatsNumThread(t *testing.T) {
+	r := NewRegistry()
+	RegisterRuntimeMemStats(r)
+	CaptureRuntimeMemStatsOnce(r)
+
+	if value := runtimeMetrics.NumThread.Value(); value < 1 {
+		t.Fatalf("got NumThread: %d, wanted at least 1", value)
+	}
+}
+
+func TestRuntimeMemStatsBlocking(t *testing.T) {
+	if g := runtime.GOMAXPROCS(0); g < 2 {
+		t.Skipf("skipping TestRuntimeMemStatsBlocking with GOMAXPROCS=%d\n", g)
+	}
+	ch := make(chan int)
+	go testRuntimeMemStatsBlocking(ch)
+	var memStats runtime.MemStats
+	t0 := time.Now()
+	runtime.ReadMemStats(&memStats)
+	t1 := time.Now()
+	t.Log("i++ during runtime.ReadMemStats:", <-ch)
+	go testRuntimeMemStatsBlocking(ch)
+	d := t1.Sub(t0)
+	t.Log(d)
+	time.Sleep(d)
+	t.Log("i++ during time.Sleep:", <-ch)
+}
+
+func testRuntimeMemStatsBlocking(ch chan int) {
+	i := 0
+	for {
+		select {
+		case ch <- i:
+			return
+		default:
+			i++
+		}
+	}
+}

metrics/sample.go

@@ -46,7 +46,7 @@ type ExpDecaySample struct {
 // NewExpDecaySample constructs a new exponentially-decaying sample with the
 // given reservoir size and alpha.
 func NewExpDecaySample(reservoirSize int, alpha float64) Sample {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilSample{}
 	}
 	s := &ExpDecaySample{
@@ -407,7 +407,7 @@ type UniformSample struct {
 // NewUniformSample constructs a new uniform sample with the given reservoir
 // size.
 func NewUniformSample(reservoirSize int) Sample {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilSample{}
 	}
 	return &UniformSample{

metrics/sample_test.go

@@ -0,0 +1,363 @@
+package metrics
+
+import (
+	"math/rand"
+	"runtime"
+	"testing"
+	"time"
+)
+
+// Benchmark{Compute,Copy}{1000,1000000} demonstrate that, even for relatively
+// expensive computations like Variance, the cost of copying the Sample, as
+// approximated by a make and copy, is much greater than the cost of the
+// computation for small samples and only slightly less for large samples.
+func BenchmarkCompute1000(b *testing.B) {
+	s := make([]int64, 1000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		SampleVariance(s)
+	}
+}
+func BenchmarkCompute1000000(b *testing.B) {
+	s := make([]int64, 1000000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		SampleVariance(s)
+	}
+}
+func BenchmarkCopy1000(b *testing.B) {
+	s := make([]int64, 1000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		sCopy := make([]int64, len(s))
+		copy(sCopy, s)
+	}
+}
+func BenchmarkCopy1000000(b *testing.B) {
+	s := make([]int64, 1000000)
+	for i := 0; i < len(s); i++ {
+		s[i] = int64(i)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		sCopy := make([]int64, len(s))
+		copy(sCopy, s)
+	}
+}
+
+func BenchmarkExpDecaySample257(b *testing.B) {
+	benchmarkSample(b, NewExpDecaySample(257, 0.015))
+}
+
+func BenchmarkExpDecaySample514(b *testing.B) {
+	benchmarkSample(b, NewExpDecaySample(514, 0.015))
+}
+
+func BenchmarkExpDecaySample1028(b *testing.B) {
+	benchmarkSample(b, NewExpDecaySample(1028, 0.015))
+}
+
+func BenchmarkUniformSample257(b *testing.B) {
+	benchmarkSample(b, NewUniformSample(257))
+}
+
+func BenchmarkUniformSample514(b *testing.B) {
+	benchmarkSample(b, NewUniformSample(514))
+}
+
+func BenchmarkUniformSample1028(b *testing.B) {
+	benchmarkSample(b, NewUniformSample(1028))
+}
+
+func TestExpDecaySample10(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 0; i < 10; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 10 != size {
+		t.Errorf("s.Count(): 10 != %v\n", size)
+	}
+	if size := s.Size(); 10 != size {
+		t.Errorf("s.Size(): 10 != %v\n", size)
+	}
+	if l := len(s.Values()); 10 != l {
+		t.Errorf("len(s.Values()): 10 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 10 || v < 0 {
+			t.Errorf("out of range [0, 10): %v\n", v)
+		}
+	}
+}
+
+func TestExpDecaySample100(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(1000, 0.01)
+	for i := 0; i < 100; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 100 != size {
+		t.Errorf("s.Count(): 100 != %v\n", size)
+	}
+	if size := s.Size(); 100 != size {
+		t.Errorf("s.Size(): 100 != %v\n", size)
+	}
+	if l := len(s.Values()); 100 != l {
+		t.Errorf("len(s.Values()): 100 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 100 || v < 0 {
+			t.Errorf("out of range [0, 100): %v\n", v)
+		}
+	}
+}
+
+func TestExpDecaySample1000(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 0; i < 1000; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 1000 != size {
+		t.Errorf("s.Count(): 1000 != %v\n", size)
+	}
+	if size := s.Size(); 100 != size {
+		t.Errorf("s.Size(): 100 != %v\n", size)
+	}
+	if l := len(s.Values()); 100 != l {
+		t.Errorf("len(s.Values()): 100 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 1000 || v < 0 {
+			t.Errorf("out of range [0, 1000): %v\n", v)
+		}
+	}
+}
+
+// This test makes sure that the sample's priority is not amplified by using
+// nanosecond duration since start rather than second duration since start.
+// The priority becomes +Inf quickly after starting if this is done,
+// effectively freezing the set of samples until a rescale step happens.
+func TestExpDecaySampleNanosecondRegression(t *testing.T) {
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 0; i < 100; i++ {
+		s.Update(10)
+	}
+	time.Sleep(1 * time.Millisecond)
+	for i := 0; i < 100; i++ {
+		s.Update(20)
+	}
+	v := s.Values()
+	avg := float64(0)
+	for i := 0; i < len(v); i++ {
+		avg += float64(v[i])
+	}
+	avg /= float64(len(v))
+	if avg > 16 || avg < 14 {
+		t.Errorf("out of range [14, 16]: %v\n", avg)
+	}
+}
+
+func TestExpDecaySampleRescale(t *testing.T) {
+	s := NewExpDecaySample(2, 0.001).(*ExpDecaySample)
+	s.update(time.Now(), 1)
+	s.update(time.Now().Add(time.Hour+time.Microsecond), 1)
+	for _, v := range s.values.Values() {
+		if v.k == 0.0 {
+			t.Fatal("v.k == 0.0")
+		}
+	}
+}
+
+func TestExpDecaySampleSnapshot(t *testing.T) {
+	now := time.Now()
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 1; i <= 10000; i++ {
+		s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i))
+	}
+	snapshot := s.Snapshot()
+	s.Update(1)
+	testExpDecaySampleStatistics(t, snapshot)
+}
+
+func TestExpDecaySampleStatistics(t *testing.T) {
+	now := time.Now()
+	rand.Seed(1)
+	s := NewExpDecaySample(100, 0.99)
+	for i := 1; i <= 10000; i++ {
+		s.(*ExpDecaySample).update(now.Add(time.Duration(i)), int64(i))
+	}
+	testExpDecaySampleStatistics(t, s)
+}
+
+func TestUniformSample(t *testing.T) {
+	rand.Seed(1)
+	s := NewUniformSample(100)
+	for i := 0; i < 1000; i++ {
+		s.Update(int64(i))
+	}
+	if size := s.Count(); 1000 != size {
+		t.Errorf("s.Count(): 1000 != %v\n", size)
+	}
+	if size := s.Size(); 100 != size {
+		t.Errorf("s.Size(): 100 != %v\n", size)
+	}
+	if l := len(s.Values()); 100 != l {
+		t.Errorf("len(s.Values()): 100 != %v\n", l)
+	}
+	for _, v := range s.Values() {
+		if v > 1000 || v < 0 {
+			t.Errorf("out of range [0, 100): %v\n", v)
+		}
+	}
+}
+
+func TestUniformSampleIncludesTail(t *testing.T) {
+	rand.Seed(1)
+	s := NewUniformSample(100)
+	max := 100
+	for i := 0; i < max; i++ {
+		s.Update(int64(i))
+	}
+	v := s.Values()
+	sum := 0
+	exp := (max - 1) * max / 2
+	for i := 0; i < len(v); i++ {
+		sum += int(v[i])
+	}
+	if exp != sum {
+		t.Errorf("sum: %v != %v\n", exp, sum)
+	}
+}
+
+func TestUniformSampleSnapshot(t *testing.T) {
+	s := NewUniformSample(100)
+	for i := 1; i <= 10000; i++ {
+		s.Update(int64(i))
+	}
+	snapshot := s.Snapshot()
+	s.Update(1)
+	testUniformSampleStatistics(t, snapshot)
+}
+
+func TestUniformSampleStatistics(t *testing.T) {
+	rand.Seed(1)
+	s := NewUniformSample(100)
+	for i := 1; i <= 10000; i++ {
+		s.Update(int64(i))
+	}
+	testUniformSampleStatistics(t, s)
+}
+
+func benchmarkSample(b *testing.B, s Sample) {
+	var memStats runtime.MemStats
+	runtime.ReadMemStats(&memStats)
+	pauseTotalNs := memStats.PauseTotalNs
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		s.Update(1)
+	}
+	b.StopTimer()
+	runtime.GC()
+	runtime.ReadMemStats(&memStats)
+	b.Logf("GC cost: %d ns/op", int(memStats.PauseTotalNs-pauseTotalNs)/b.N)
+}
+
+func testExpDecaySampleStatistics(t *testing.T, s Sample) {
+	if count := s.Count(); 10000 != count {
+		t.Errorf("s.Count(): 10000 != %v\n", count)
+	}
+	if min := s.Min(); 107 != min {
+		t.Errorf("s.Min(): 107 != %v\n", min)
+	}
+	if max := s.Max(); 10000 != max {
+		t.Errorf("s.Max(): 10000 != %v\n", max)
+	}
+	if mean := s.Mean(); 4965.98 != mean {
+		t.Errorf("s.Mean(): 4965.98 != %v\n", mean)
+	}
+	if stdDev := s.StdDev(); 2959.825156930727 != stdDev {
+		t.Errorf("s.StdDev(): 2959.825156930727 != %v\n", stdDev)
+	}
+	ps := s.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 4615 != ps[0] {
+		t.Errorf("median: 4615 != %v\n", ps[0])
+	}
+	if 7672 != ps[1] {
+		t.Errorf("75th percentile: 7672 != %v\n", ps[1])
+	}
+	if 9998.99 != ps[2] {
+		t.Errorf("99th percentile: 9998.99 != %v\n", ps[2])
+	}
+}
+
+func testUniformSampleStatistics(t *testing.T, s Sample) {
+	if count := s.Count(); 10000 != count {
+		t.Errorf("s.Count(): 10000 != %v\n", count)
+	}
+	if min := s.Min(); 37 != min {
+		t.Errorf("s.Min(): 37 != %v\n", min)
+	}
+	if max := s.Max(); 9989 != max {
+		t.Errorf("s.Max(): 9989 != %v\n", max)
+	}
+	if mean := s.Mean(); 4748.14 != mean {
+		t.Errorf("s.Mean(): 4748.14 != %v\n", mean)
+	}
+	if stdDev := s.StdDev(); 2826.684117548333 != stdDev {
+		t.Errorf("s.StdDev(): 2826.684117548333 != %v\n", stdDev)
+	}
+	ps := s.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 4599 != ps[0] {
+		t.Errorf("median: 4599 != %v\n", ps[0])
+	}
+	if 7380.5 != ps[1] {
+		t.Errorf("75th percentile: 7380.5 != %v\n", ps[1])
+	}
+	if 9986.429999999998 != ps[2] {
+		t.Errorf("99th percentile: 9986.429999999998 != %v\n", ps[2])
+	}
+}
+
+// TestUniformSampleConcurrentUpdateCount would expose data race problems with
+// concurrent Update and Count calls on Sample when test is called with -race
+// argument
+func TestUniformSampleConcurrentUpdateCount(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+	s := NewUniformSample(100)
+	for i := 0; i < 100; i++ {
+		s.Update(int64(i))
+	}
+	quit := make(chan struct{})
+	go func() {
+		t := time.NewTicker(10 * time.Millisecond)
+		for {
+			select {
+			case <-t.C:
+				s.Update(rand.Int63())
+			case <-quit:
+				t.Stop()
+				return
+			}
+		}
+	}()
+	for i := 0; i < 1000; i++ {
+		s.Count()
+		time.Sleep(5 * time.Millisecond)
+	}
+	quit <- struct{}{}
+}

metrics/syslog.go

@@ -11,7 +11,7 @@ import (
 // Output each metric in the given registry to syslog periodically using
 // the given syslogger.
 func Syslog(r Registry, d time.Duration, w *syslog.Writer) {
-	for _ = range time.Tick(d) {
+	for range time.Tick(d) {
 		r.Each(func(name string, i interface{}) {
 			switch metric := i.(type) {
 			case Counter:

metrics/timer.go

@@ -19,6 +19,7 @@ type Timer interface {
 	RateMean() float64
 	Snapshot() Timer
 	StdDev() float64
+	Stop()
 	Sum() int64
 	Time(func())
 	Update(time.Duration)
@@ -28,6 +29,8 @@ type Timer interface {
 
 // GetOrRegisterTimer returns an existing Timer or constructs and registers a
 // new StandardTimer.
+// Be sure to unregister the meter from the registry once it is of no use to
+// allow for garbage collection.
 func GetOrRegisterTimer(name string, r Registry) Timer {
 	if nil == r {
 		r = DefaultRegistry
@@ -36,8 +39,9 @@ func GetOrRegisterTimer(name string, r Registry) Timer {
 }
 
 // NewCustomTimer constructs a new StandardTimer from a Histogram and a Meter.
+// Be sure to call Stop() once the timer is of no use to allow for garbage collection.
 func NewCustomTimer(h Histogram, m Meter) Timer {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilTimer{}
 	}
 	return &StandardTimer{
@@ -47,6 +51,8 @@ func NewCustomTimer(h Histogram, m Meter) Timer {
 }
 
 // NewRegisteredTimer constructs and registers a new StandardTimer.
+// Be sure to unregister the meter from the registry once it is of no use to
+// allow for garbage collection.
 func NewRegisteredTimer(name string, r Registry) Timer {
 	c := NewTimer()
 	if nil == r {
@@ -58,8 +64,9 @@ func NewRegisteredTimer(name string, r Registry) Timer {
 
 // NewTimer constructs a new StandardTimer using an exponentially-decaying
 // sample with the same reservoir size and alpha as UNIX load averages.
+// Be sure to call Stop() once the timer is of no use to allow for garbage collection.
 func NewTimer() Timer {
-	if UseNilMetrics {
+	if !Enabled {
 		return NilTimer{}
 	}
 	return &StandardTimer{
@@ -112,6 +119,9 @@ func (NilTimer) Snapshot() Timer { return NilTimer{} }
 // StdDev is a no-op.
 func (NilTimer) StdDev() float64 { return 0.0 }
 
+// Stop is a no-op.
+func (NilTimer) Stop() {}
+
 // Sum is a no-op.
 func (NilTimer) Sum() int64 { return 0 }
 
@@ -201,6 +211,11 @@ func (t *StandardTimer) StdDev() float64 {
 	return t.histogram.StdDev()
 }
 
+// Stop stops the meter.
+func (t *StandardTimer) Stop() {
+	t.meter.Stop()
+}
+
 // Sum returns the sum in the sample.
 func (t *StandardTimer) Sum() int64 {
 	return t.histogram.Sum()
@@ -288,6 +303,9 @@ func (t *TimerSnapshot) Snapshot() Timer { return t }
 // was taken.
 func (t *TimerSnapshot) StdDev() float64 { return t.histogram.StdDev() }
 
+// Stop is a no-op.
+func (t *TimerSnapshot) Stop() {}
+
 // Sum returns the sum at the time the snapshot was taken.
 func (t *TimerSnapshot) Sum() int64 { return t.histogram.Sum() }
 

metrics/timer_test.go

@@ -0,0 +1,101 @@
+package metrics
+
+import (
+	"fmt"
+	"math"
+	"testing"
+	"time"
+)
+
+func BenchmarkTimer(b *testing.B) {
+	tm := NewTimer()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		tm.Update(1)
+	}
+}
+
+func TestGetOrRegisterTimer(t *testing.T) {
+	r := NewRegistry()
+	NewRegisteredTimer("foo", r).Update(47)
+	if tm := GetOrRegisterTimer("foo", r); 1 != tm.Count() {
+		t.Fatal(tm)
+	}
+}
+
+func TestTimerExtremes(t *testing.T) {
+	tm := NewTimer()
+	tm.Update(math.MaxInt64)
+	tm.Update(0)
+	if stdDev := tm.StdDev(); 4.611686018427388e+18 != stdDev {
+		t.Errorf("tm.StdDev(): 4.611686018427388e+18 != %v\n", stdDev)
+	}
+}
+
+func TestTimerStop(t *testing.T) {
+	l := len(arbiter.meters)
+	tm := NewTimer()
+	if len(arbiter.meters) != l+1 {
+		t.Errorf("arbiter.meters: %d != %d\n", l+1, len(arbiter.meters))
+	}
+	tm.Stop()
+	if len(arbiter.meters) != l {
+		t.Errorf("arbiter.meters: %d != %d\n", l, len(arbiter.meters))
+	}
+}
+
+func TestTimerFunc(t *testing.T) {
+	tm := NewTimer()
+	tm.Time(func() { time.Sleep(50e6) })
+	if max := tm.Max(); 45e6 > max || max > 55e6 {
+		t.Errorf("tm.Max(): 45e6 > %v || %v > 55e6\n", max, max)
+	}
+}
+
+func TestTimerZero(t *testing.T) {
+	tm := NewTimer()
+	if count := tm.Count(); 0 != count {
+		t.Errorf("tm.Count(): 0 != %v\n", count)
+	}
+	if min := tm.Min(); 0 != min {
+		t.Errorf("tm.Min(): 0 != %v\n", min)
+	}
+	if max := tm.Max(); 0 != max {
+		t.Errorf("tm.Max(): 0 != %v\n", max)
+	}
+	if mean := tm.Mean(); 0.0 != mean {
+		t.Errorf("tm.Mean(): 0.0 != %v\n", mean)
+	}
+	if stdDev := tm.StdDev(); 0.0 != stdDev {
+		t.Errorf("tm.StdDev(): 0.0 != %v\n", stdDev)
+	}
+	ps := tm.Percentiles([]float64{0.5, 0.75, 0.99})
+	if 0.0 != ps[0] {
+		t.Errorf("median: 0.0 != %v\n", ps[0])
+	}
+	if 0.0 != ps[1] {
+		t.Errorf("75th percentile: 0.0 != %v\n", ps[1])
+	}
+	if 0.0 != ps[2] {
+		t.Errorf("99th percentile: 0.0 != %v\n", ps[2])
+	}
+	if rate1 := tm.Rate1(); 0.0 != rate1 {
+		t.Errorf("tm.Rate1(): 0.0 != %v\n", rate1)
+	}
+	if rate5 := tm.Rate5(); 0.0 != rate5 {
+		t.Errorf("tm.Rate5(): 0.0 != %v\n", rate5)
+	}
+	if rate15 := tm.Rate15(); 0.0 != rate15 {
+		t.Errorf("tm.Rate15(): 0.0 != %v\n", rate15)
+	}
+	if rateMean := tm.RateMean(); 0.0 != rateMean {
+		t.Errorf("tm.RateMean(): 0.0 != %v\n", rateMean)
+	}
+}
+
+func ExampleGetOrRegisterTimer() {
+	m := "account.create.latency"
+	t := GetOrRegisterTimer(m, nil)
+	t.Update(47)
+	fmt.Println(t.Max()) // Output: 47
+}

metrics/validate.sh

@@ -7,4 +7,4 @@ GOFMT_LINES=`gofmt -l . | wc -l | xargs`
 test $GOFMT_LINES -eq 0 || echo "gofmt needs to be run, ${GOFMT_LINES} files have issues"
 
 # run the tests for the root package
-go test .
+go test -race .

metrics/writer.go

@@ -10,7 +10,7 @@ import (
 // Write sorts writes each metric in the given registry periodically to the
 // given io.Writer.
 func Write(r Registry, d time.Duration, w io.Writer) {
-	for _ = range time.Tick(d) {
+	for range time.Tick(d) {
 		WriteOnce(r, w)
 	}
 }

metrics/writer_test.go

@@ -0,0 +1,22 @@
+package metrics
+
+import (
+	"sort"
+	"testing"
+)
+
+func TestMetricsSorting(t *testing.T) {
+	var namedMetrics = namedMetricSlice{
+		{name: "zzz"},
+		{name: "bbb"},
+		{name: "fff"},
+		{name: "ggg"},
+	}
+
+	sort.Sort(namedMetrics)
+	for i, name := range []string{"bbb", "fff", "ggg", "zzz"} {
+		if namedMetrics[i].name != name {
+			t.Fail()
+		}
+	}
+}

node/api.go

@@ -24,10 +24,10 @@ import (
 
 	"github.com/EthereumCommonwealth/go-callisto/common/hexutil"
 	"github.com/EthereumCommonwealth/go-callisto/crypto"
+	"github.com/EthereumCommonwealth/go-callisto/metrics"
 	"github.com/EthereumCommonwealth/go-callisto/p2p"
 	"github.com/EthereumCommonwealth/go-callisto/p2p/discover"
 	"github.com/EthereumCommonwealth/go-callisto/rpc"
-	"github.com/rcrowley/go-metrics"
 )
 
 // PrivateAdminAPI is the collection of administrative API methods exposed only

p2p/message.go

@@ -22,8 +22,6 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"net"
-	"sync"
 	"sync/atomic"
 	"time"
 
@@ -112,30 +110,6 @@ func SendItems(w MsgWriter, msgcode uint64, elems ...interface{}) error {
 	return Send(w, msgcode, elems)
 }
 
-// netWrapper wraps a MsgReadWriter with locks around
-// ReadMsg/WriteMsg and applies read/write deadlines.
-type netWrapper struct {
-	rmu, wmu sync.Mutex
-
-	rtimeout, wtimeout time.Duration
-	conn               net.Conn
-	wrapped            MsgReadWriter
-}
-
-func (rw *netWrapper) ReadMsg() (Msg, error) {
-	rw.rmu.Lock()
-	defer rw.rmu.Unlock()
-	rw.conn.SetReadDeadline(time.Now().Add(rw.rtimeout))
-	return rw.wrapped.ReadMsg()
-}
-
-func (rw *netWrapper) WriteMsg(msg Msg) error {
-	rw.wmu.Lock()
-	defer rw.wmu.Unlock()
-	rw.conn.SetWriteDeadline(time.Now().Add(rw.wtimeout))
-	return rw.wrapped.WriteMsg(msg)
-}
-
 // eofSignal wraps a reader with eof signaling. the eof channel is
 // closed when the wrapped reader returns an error or when count bytes
 // have been read.

p2p/metrics.go

@@ -25,10 +25,10 @@ import (
 )
 
 var (
-	ingressConnectMeter = metrics.NewMeter("p2p/InboundConnects")
+	ingressConnectMeter = metrics.NewRegisteredMeter("p2p/InboundConnects", nil)
-	ingressTrafficMeter = metrics.NewMeter("p2p/InboundTraffic")
+	ingressTrafficMeter = metrics.NewRegisteredMeter("p2p/InboundTraffic", nil)
-	egressConnectMeter  = metrics.NewMeter("p2p/OutboundConnects")
+	egressConnectMeter  = metrics.NewRegisteredMeter("p2p/OutboundConnects", nil)
-	egressTrafficMeter  = metrics.NewMeter("p2p/OutboundTraffic")
+	egressTrafficMeter  = metrics.NewRegisteredMeter("p2p/OutboundTraffic", nil)
 )
 
 // meteredConn is a wrapper around a network TCP connection that meters both the

p2p/protocols/protocol.go

@@ -0,0 +1,311 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+/*
+Package protocols is an extension to p2p. It offers a user friendly simple way to define
+devp2p subprotocols by abstracting away code standardly shared by protocols.
+
+* automate assigments of code indexes to messages
+* automate RLP decoding/encoding based on reflecting
+* provide the forever loop to read incoming messages
+* standardise error handling related to communication
+* standardised	handshake negotiation
+* TODO: automatic generation of wire protocol specification for peers
+
+*/
+package protocols
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"sync"
+
+	"github.com/EthereumCommonwealth/go-callisto/p2p"
+)
+
+// error codes used by this  protocol scheme
+const (
+	ErrMsgTooLong = iota
+	ErrDecode
+	ErrWrite
+	ErrInvalidMsgCode
+	ErrInvalidMsgType
+	ErrHandshake
+	ErrNoHandler
+	ErrHandler
+)
+
+// error description strings associated with the codes
+var errorToString = map[int]string{
+	ErrMsgTooLong:     "Message too long",
+	ErrDecode:         "Invalid message (RLP error)",
+	ErrWrite:          "Error sending message",
+	ErrInvalidMsgCode: "Invalid message code",
+	ErrInvalidMsgType: "Invalid message type",
+	ErrHandshake:      "Handshake error",
+	ErrNoHandler:      "No handler registered error",
+	ErrHandler:        "Message handler error",
+}
+
+/*
+Error implements the standard go error interface.
+Use:
+
+  errorf(code, format, params ...interface{})
+
+Prints as:
+
+ <description>: <details>
+
+where description is given by code in errorToString
+and details is fmt.Sprintf(format, params...)
+
+exported field Code can be checked
+*/
+type Error struct {
+	Code    int
+	message string
+	format  string
+	params  []interface{}
+}
+
+func (e Error) Error() (message string) {
+	if len(e.message) == 0 {
+		name, ok := errorToString[e.Code]
+		if !ok {
+			panic("invalid message code")
+		}
+		e.message = name
+		if e.format != "" {
+			e.message += ": " + fmt.Sprintf(e.format, e.params...)
+		}
+	}
+	return e.message
+}
+
+func errorf(code int, format string, params ...interface{}) *Error {
+	return &Error{
+		Code:   code,
+		format: format,
+		params: params,
+	}
+}
+
+// Spec is a protocol specification including its name and version as well as
+// the types of messages which are exchanged
+type Spec struct {
+	// Name is the name of the protocol, often a three-letter word
+	Name string
+
+	// Version is the version number of the protocol
+	Version uint
+
+	// MaxMsgSize is the maximum accepted length of the message payload
+	MaxMsgSize uint32
+
+	// Messages is a list of message data types which this protocol uses, with
+	// each message type being sent with its array index as the code (so
+	// [&foo{}, &bar{}, &baz{}] would send foo, bar and baz with codes
+	// 0, 1 and 2 respectively)
+	// each message must have a single unique data type
+	Messages []interface{}
+
+	initOnce sync.Once
+	codes    map[reflect.Type]uint64
+	types    map[uint64]reflect.Type
+}
+
+func (s *Spec) init() {
+	s.initOnce.Do(func() {
+		s.codes = make(map[reflect.Type]uint64, len(s.Messages))
+		s.types = make(map[uint64]reflect.Type, len(s.Messages))
+		for i, msg := range s.Messages {
+			code := uint64(i)
+			typ := reflect.TypeOf(msg)
+			if typ.Kind() == reflect.Ptr {
+				typ = typ.Elem()
+			}
+			s.codes[typ] = code
+			s.types[code] = typ
+		}
+	})
+}
+
+// Length returns the number of message types in the protocol
+func (s *Spec) Length() uint64 {
+	return uint64(len(s.Messages))
+}
+
+// GetCode returns the message code of a type, and boolean second argument is
+// false if the message type is not found
+func (s *Spec) GetCode(msg interface{}) (uint64, bool) {
+	s.init()
+	typ := reflect.TypeOf(msg)
+	if typ.Kind() == reflect.Ptr {
+		typ = typ.Elem()
+	}
+	code, ok := s.codes[typ]
+	return code, ok
+}
+
+// NewMsg construct a new message type given the code
+func (s *Spec) NewMsg(code uint64) (interface{}, bool) {
+	s.init()
+	typ, ok := s.types[code]
+	if !ok {
+		return nil, false
+	}
+	return reflect.New(typ).Interface(), true
+}
+
+// Peer represents a remote peer or protocol instance that is running on a peer connection with
+// a remote peer
+type Peer struct {
+	*p2p.Peer                   // the p2p.Peer object representing the remote
+	rw        p2p.MsgReadWriter // p2p.MsgReadWriter to send messages to and read messages from
+	spec      *Spec
+}
+
+// NewPeer constructs a new peer
+// this constructor is called by the p2p.Protocol#Run function
+// the first two arguments are the arguments passed to p2p.Protocol.Run function
+// the third argument is the Spec describing the protocol
+func NewPeer(p *p2p.Peer, rw p2p.MsgReadWriter, spec *Spec) *Peer {
+	return &Peer{
+		Peer: p,
+		rw:   rw,
+		spec: spec,
+	}
+}
+
+// Run starts the forever loop that handles incoming messages
+// called within the p2p.Protocol#Run function
+// the handler argument is a function which is called for each message received
+// from the remote peer, a returned error causes the loop to exit
+// resulting in disconnection
+func (p *Peer) Run(handler func(msg interface{}) error) error {
+	for {
+		if err := p.handleIncoming(handler); err != nil {
+			return err
+		}
+	}
+}
+
+// Drop disconnects a peer.
+// TODO: may need to implement protocol drop only? don't want to kick off the peer
+// if they are useful for other protocols
+func (p *Peer) Drop(err error) {
+	p.Disconnect(p2p.DiscSubprotocolError)
+}
+
+// Send takes a message, encodes it in RLP, finds the right message code and sends the
+// message off to the peer
+// this low level call will be wrapped by libraries providing routed or broadcast sends
+// but often just used to forward and push messages to directly connected peers
+func (p *Peer) Send(msg interface{}) error {
+	code, found := p.spec.GetCode(msg)
+	if !found {
+		return errorf(ErrInvalidMsgType, "%v", code)
+	}
+	return p2p.Send(p.rw, code, msg)
+}
+
+// handleIncoming(code)
+// is called each cycle of the main forever loop that dispatches incoming messages
+// if this returns an error the loop returns and the peer is disconnected with the error
+// this generic handler
+// * checks message size,
+// * checks for out-of-range message codes,
+// * handles decoding with reflection,
+// * call handlers as callbacks
+func (p *Peer) handleIncoming(handle func(msg interface{}) error) error {
+	msg, err := p.rw.ReadMsg()
+	if err != nil {
+		return err
+	}
+	// make sure that the payload has been fully consumed
+	defer msg.Discard()
+
+	if msg.Size > p.spec.MaxMsgSize {
+		return errorf(ErrMsgTooLong, "%v > %v", msg.Size, p.spec.MaxMsgSize)
+	}
+
+	val, ok := p.spec.NewMsg(msg.Code)
+	if !ok {
+		return errorf(ErrInvalidMsgCode, "%v", msg.Code)
+	}
+	if err := msg.Decode(val); err != nil {
+		return errorf(ErrDecode, "<= %v: %v", msg, err)
+	}
+
+	// call the registered handler callbacks
+	// a registered callback take the decoded message as argument as an interface
+	// which the handler is supposed to cast to the appropriate type
+	// it is entirely safe not to check the cast in the handler since the handler is
+	// chosen based on the proper type in the first place
+	if err := handle(val); err != nil {
+		return errorf(ErrHandler, "(msg code %v): %v", msg.Code, err)
+	}
+	return nil
+}
+
+// Handshake negotiates a handshake on the peer connection
+// * arguments
+//   * context
+//   * the local handshake to be sent to the remote peer
+//   * funcion to be called on the remote handshake (can be nil)
+// * expects a remote handshake back of the same type
+// * the dialing peer needs to send the handshake first and then waits for remote
+// * the listening peer waits for the remote handshake and then sends it
+// returns the remote handshake and an error
+func (p *Peer) Handshake(ctx context.Context, hs interface{}, verify func(interface{}) error) (rhs interface{}, err error) {
+	if _, ok := p.spec.GetCode(hs); !ok {
+		return nil, errorf(ErrHandshake, "unknown handshake message type: %T", hs)
+	}
+	errc := make(chan error, 2)
+	handle := func(msg interface{}) error {
+		rhs = msg
+		if verify != nil {
+			return verify(rhs)
+		}
+		return nil
+	}
+	send := func() { errc <- p.Send(hs) }
+	receive := func() { errc <- p.handleIncoming(handle) }
+
+	go func() {
+		if p.Inbound() {
+			receive()
+			send()
+		} else {
+			send()
+			receive()
+		}
+	}()
+
+	for i := 0; i < 2; i++ {
+		select {
+		case err = <-errc:
+		case <-ctx.Done():
+			err = ctx.Err()
+		}
+		if err != nil {
+			return nil, errorf(ErrHandshake, err.Error())
+		}
+	}
+	return rhs, nil
+}

p2p/protocols/protocol_test.go

@@ -0,0 +1,389 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package protocols
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/EthereumCommonwealth/go-callisto/p2p"
+	"github.com/EthereumCommonwealth/go-callisto/p2p/discover"
+	"github.com/EthereumCommonwealth/go-callisto/p2p/simulations/adapters"
+	p2ptest "github.com/EthereumCommonwealth/go-callisto/p2p/testing"
+)
+
+// handshake message type
+type hs0 struct {
+	C uint
+}
+
+// message to kill/drop the peer with nodeID
+type kill struct {
+	C discover.NodeID
+}
+
+// message to drop connection
+type drop struct {
+}
+
+/// protoHandshake represents module-independent aspects of the protocol and is
+// the first message peers send and receive as part the initial exchange
+type protoHandshake struct {
+	Version   uint   // local and remote peer should have identical version
+	NetworkID string // local and remote peer should have identical network id
+}
+
+// checkProtoHandshake verifies local and remote protoHandshakes match
+func checkProtoHandshake(testVersion uint, testNetworkID string) func(interface{}) error {
+	return func(rhs interface{}) error {
+		remote := rhs.(*protoHandshake)
+		if remote.NetworkID != testNetworkID {
+			return fmt.Errorf("%s (!= %s)", remote.NetworkID, testNetworkID)
+		}
+
+		if remote.Version != testVersion {
+			return fmt.Errorf("%d (!= %d)", remote.Version, testVersion)
+		}
+		return nil
+	}
+}
+
+// newProtocol sets up a protocol
+// the run function here demonstrates a typical protocol using peerPool, handshake
+// and messages registered to handlers
+func newProtocol(pp *p2ptest.TestPeerPool) func(*p2p.Peer, p2p.MsgReadWriter) error {
+	spec := &Spec{
+		Name:       "test",
+		Version:    42,
+		MaxMsgSize: 10 * 1024,
+		Messages: []interface{}{
+			protoHandshake{},
+			hs0{},
+			kill{},
+			drop{},
+		},
+	}
+	return func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
+		peer := NewPeer(p, rw, spec)
+
+		// initiate one-off protohandshake and check validity
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		phs := &protoHandshake{42, "420"}
+		hsCheck := checkProtoHandshake(phs.Version, phs.NetworkID)
+		_, err := peer.Handshake(ctx, phs, hsCheck)
+		if err != nil {
+			return err
+		}
+
+		lhs := &hs0{42}
+		// module handshake demonstrating a simple repeatable exchange of same-type message
+		hs, err := peer.Handshake(ctx, lhs, nil)
+		if err != nil {
+			return err
+		}
+
+		if rmhs := hs.(*hs0); rmhs.C > lhs.C {
+			return fmt.Errorf("handshake mismatch remote %v > local %v", rmhs.C, lhs.C)
+		}
+
+		handle := func(msg interface{}) error {
+			switch msg := msg.(type) {
+
+			case *protoHandshake:
+				return errors.New("duplicate handshake")
+
+			case *hs0:
+				rhs := msg
+				if rhs.C > lhs.C {
+					return fmt.Errorf("handshake mismatch remote %v > local %v", rhs.C, lhs.C)
+				}
+				lhs.C += rhs.C
+				return peer.Send(lhs)
+
+			case *kill:
+				// demonstrates use of peerPool, killing another peer connection as a response to a message
+				id := msg.C
+				pp.Get(id).Drop(errors.New("killed"))
+				return nil
+
+			case *drop:
+				// for testing we can trigger self induced disconnect upon receiving drop message
+				return errors.New("dropped")
+
+			default:
+				return fmt.Errorf("unknown message type: %T", msg)
+			}
+		}
+
+		pp.Add(peer)
+		defer pp.Remove(peer)
+		return peer.Run(handle)
+	}
+}
+
+func protocolTester(t *testing.T, pp *p2ptest.TestPeerPool) *p2ptest.ProtocolTester {
+	conf := adapters.RandomNodeConfig()
+	return p2ptest.NewProtocolTester(t, conf.ID, 2, newProtocol(pp))
+}
+
+func protoHandshakeExchange(id discover.NodeID, proto *protoHandshake) []p2ptest.Exchange {
+
+	return []p2ptest.Exchange{
+		{
+			Expects: []p2ptest.Expect{
+				{
+					Code: 0,
+					Msg:  &protoHandshake{42, "420"},
+					Peer: id,
+				},
+			},
+		},
+		{
+			Triggers: []p2ptest.Trigger{
+				{
+					Code: 0,
+					Msg:  proto,
+					Peer: id,
+				},
+			},
+		},
+	}
+}
+
+func runProtoHandshake(t *testing.T, proto *protoHandshake, errs ...error) {
+	pp := p2ptest.NewTestPeerPool()
+	s := protocolTester(t, pp)
+	// TODO: make this more than one handshake
+	id := s.IDs[0]
+	if err := s.TestExchanges(protoHandshakeExchange(id, proto)...); err != nil {
+		t.Fatal(err)
+	}
+	var disconnects []*p2ptest.Disconnect
+	for i, err := range errs {
+		disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err})
+	}
+	if err := s.TestDisconnected(disconnects...); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestProtoHandshakeVersionMismatch(t *testing.T) {
+	runProtoHandshake(t, &protoHandshake{41, "420"}, errorf(ErrHandshake, errorf(ErrHandler, "(msg code 0): 41 (!= 42)").Error()))
+}
+
+func TestProtoHandshakeNetworkIDMismatch(t *testing.T) {
+	runProtoHandshake(t, &protoHandshake{42, "421"}, errorf(ErrHandshake, errorf(ErrHandler, "(msg code 0): 421 (!= 420)").Error()))
+}
+
+func TestProtoHandshakeSuccess(t *testing.T) {
+	runProtoHandshake(t, &protoHandshake{42, "420"})
+}
+
+func moduleHandshakeExchange(id discover.NodeID, resp uint) []p2ptest.Exchange {
+
+	return []p2ptest.Exchange{
+		{
+			Expects: []p2ptest.Expect{
+				{
+					Code: 1,
+					Msg:  &hs0{42},
+					Peer: id,
+				},
+			},
+		},
+		{
+			Triggers: []p2ptest.Trigger{
+				{
+					Code: 1,
+					Msg:  &hs0{resp},
+					Peer: id,
+				},
+			},
+		},
+	}
+}
+
+func runModuleHandshake(t *testing.T, resp uint, errs ...error) {
+	pp := p2ptest.NewTestPeerPool()
+	s := protocolTester(t, pp)
+	id := s.IDs[0]
+	if err := s.TestExchanges(protoHandshakeExchange(id, &protoHandshake{42, "420"})...); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.TestExchanges(moduleHandshakeExchange(id, resp)...); err != nil {
+		t.Fatal(err)
+	}
+	var disconnects []*p2ptest.Disconnect
+	for i, err := range errs {
+		disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err})
+	}
+	if err := s.TestDisconnected(disconnects...); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestModuleHandshakeError(t *testing.T) {
+	runModuleHandshake(t, 43, fmt.Errorf("handshake mismatch remote 43 > local 42"))
+}
+
+func TestModuleHandshakeSuccess(t *testing.T) {
+	runModuleHandshake(t, 42)
+}
+
+// testing complex interactions over multiple peers, relaying, dropping
+func testMultiPeerSetup(a, b discover.NodeID) []p2ptest.Exchange {
+
+	return []p2ptest.Exchange{
+		{
+			Label: "primary handshake",
+			Expects: []p2ptest.Expect{
+				{
+					Code: 0,
+					Msg:  &protoHandshake{42, "420"},
+					Peer: a,
+				},
+				{
+					Code: 0,
+					Msg:  &protoHandshake{42, "420"},
+					Peer: b,
+				},
+			},
+		},
+		{
+			Label: "module handshake",
+			Triggers: []p2ptest.Trigger{
+				{
+					Code: 0,
+					Msg:  &protoHandshake{42, "420"},
+					Peer: a,
+				},
+				{
+					Code: 0,
+					Msg:  &protoHandshake{42, "420"},
+					Peer: b,
+				},
+			},
+			Expects: []p2ptest.Expect{
+				{
+					Code: 1,
+					Msg:  &hs0{42},
+					Peer: a,
+				},
+				{
+					Code: 1,
+					Msg:  &hs0{42},
+					Peer: b,
+				},
+			},
+		},
+
+		{Label: "alternative module handshake", Triggers: []p2ptest.Trigger{{Code: 1, Msg: &hs0{41}, Peer: a},
+			{Code: 1, Msg: &hs0{41}, Peer: b}}},
+		{Label: "repeated module handshake", Triggers: []p2ptest.Trigger{{Code: 1, Msg: &hs0{1}, Peer: a}}},
+		{Label: "receiving repeated module handshake", Expects: []p2ptest.Expect{{Code: 1, Msg: &hs0{43}, Peer: a}}}}
+}
+
+func runMultiplePeers(t *testing.T, peer int, errs ...error) {
+	pp := p2ptest.NewTestPeerPool()
+	s := protocolTester(t, pp)
+
+	if err := s.TestExchanges(testMultiPeerSetup(s.IDs[0], s.IDs[1])...); err != nil {
+		t.Fatal(err)
+	}
+	// after some exchanges of messages, we can test state changes
+	// here this is simply demonstrated by the peerPool
+	// after the handshake negotiations peers must be added to the pool
+	// time.Sleep(1)
+	tick := time.NewTicker(10 * time.Millisecond)
+	timeout := time.NewTimer(1 * time.Second)
+WAIT:
+	for {
+		select {
+		case <-tick.C:
+			if pp.Has(s.IDs[0]) {
+				break WAIT
+			}
+		case <-timeout.C:
+			t.Fatal("timeout")
+		}
+	}
+	if !pp.Has(s.IDs[1]) {
+		t.Fatalf("missing peer test-1: %v (%v)", pp, s.IDs)
+	}
+
+	// peer 0 sends kill request for peer with index <peer>
+	err := s.TestExchanges(p2ptest.Exchange{
+		Triggers: []p2ptest.Trigger{
+			{
+				Code: 2,
+				Msg:  &kill{s.IDs[peer]},
+				Peer: s.IDs[0],
+			},
+		},
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// the peer not killed sends a drop request
+	err = s.TestExchanges(p2ptest.Exchange{
+		Triggers: []p2ptest.Trigger{
+			{
+				Code: 3,
+				Msg:  &drop{},
+				Peer: s.IDs[(peer+1)%2],
+			},
+		},
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check the actual discconnect errors on the individual peers
+	var disconnects []*p2ptest.Disconnect
+	for i, err := range errs {
+		disconnects = append(disconnects, &p2ptest.Disconnect{Peer: s.IDs[i], Error: err})
+	}
+	if err := s.TestDisconnected(disconnects...); err != nil {
+		t.Fatal(err)
+	}
+	// test if disconnected peers have been removed from peerPool
+	if pp.Has(s.IDs[peer]) {
+		t.Fatalf("peer test-%v not dropped: %v (%v)", peer, pp, s.IDs)
+	}
+
+}
+
+func TestMultiplePeersDropSelf(t *testing.T) {
+	runMultiplePeers(t, 0,
+		fmt.Errorf("subprotocol error"),
+		fmt.Errorf("Message handler error: (msg code 3): dropped"),
+	)
+}
+
+func TestMultiplePeersDropOther(t *testing.T) {
+	runMultiplePeers(t, 1,
+		fmt.Errorf("Message handler error: (msg code 3): dropped"),
+		fmt.Errorf("subprotocol error"),
+	)
+}

p2p/rlpx.go

@@ -108,8 +108,14 @@ func (t *rlpx) close(err error) {
 	// Tell the remote end why we're disconnecting if possible.
 	if t.rw != nil {
 		if r, ok := err.(DiscReason); ok && r != DiscNetworkError {
-			t.fd.SetWriteDeadline(time.Now().Add(discWriteTimeout))
+			// rlpx tries to send DiscReason to disconnected peer
-			SendItems(t.rw, discMsg, r)
+			// if the connection is net.Pipe (in-memory simulation)
+			// it hangs forever, since net.Pipe does not implement
+			// a write deadline. Because of this only try to send
+			// the disconnect reason message if there is no error.
+			if err := t.fd.SetWriteDeadline(time.Now().Add(discWriteTimeout)); err == nil {
+				SendItems(t.rw, discMsg, r)
+			}
 		}
 	}
 	t.fd.Close()

p2p/rlpx_test.go

@@ -156,14 +156,18 @@ func TestProtocolHandshake(t *testing.T) {
 		node1   = &discover.Node{ID: discover.PubkeyID(&prv1.PublicKey), IP: net.IP{5, 6, 7, 8}, TCP: 44}
 		hs1     = &protoHandshake{Version: 3, ID: node1.ID, Caps: []Cap{{"c", 1}, {"d", 3}}}
 
-		fd0, fd1 = net.Pipe()
+		wg sync.WaitGroup
-		wg       sync.WaitGroup
 	)
 
+	fd0, fd1, err := tcpPipe()
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	wg.Add(2)
 	go func() {
 		defer wg.Done()
-		defer fd1.Close()
+		defer fd0.Close()
 		rlpx := newRLPX(fd0)
 		remid, err := rlpx.doEncHandshake(prv0, node1)
 		if err != nil {
@@ -597,3 +601,31 @@ func TestHandshakeForwardCompatibility(t *testing.T) {
 		t.Errorf("ingress-mac('foo') mismatch:\ngot %x\nwant %x", fooIngressHash, wantFooIngressHash)
 	}
 }
+
+// tcpPipe creates an in process full duplex pipe based on a localhost TCP socket
+func tcpPipe() (net.Conn, net.Conn, error) {
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		return nil, nil, err
+	}
+	defer l.Close()
+
+	var aconn net.Conn
+	aerr := make(chan error, 1)
+	go func() {
+		var err error
+		aconn, err = l.Accept()
+		aerr <- err
+	}()
+
+	dconn, err := net.Dial("tcp", l.Addr().String())
+	if err != nil {
+		<-aerr
+		return nil, nil, err
+	}
+	if err := <-aerr; err != nil {
+		dconn.Close()
+		return nil, nil, err
+	}
+	return aconn, dconn, nil
+}

p2p/server.go

@@ -36,9 +36,7 @@ import (
 )
 
 const (
-	defaultDialTimeout      = 15 * time.Second
+	defaultDialTimeout = 15 * time.Second
-	refreshPeersInterval    = 30 * time.Second
-	staticPeerCheckInterval = 15 * time.Second
 
 	// Connectivity defaults.
 	maxActiveDialTasks     = 16

p2p/testing/peerpool.go

@@ -0,0 +1,67 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package testing
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/EthereumCommonwealth/go-callisto/log"
+	"github.com/EthereumCommonwealth/go-callisto/p2p/discover"
+)
+
+type TestPeer interface {
+	ID() discover.NodeID
+	Drop(error)
+}
+
+// TestPeerPool is an example peerPool to demonstrate registration of peer connections
+type TestPeerPool struct {
+	lock  sync.Mutex
+	peers map[discover.NodeID]TestPeer
+}
+
+func NewTestPeerPool() *TestPeerPool {
+	return &TestPeerPool{peers: make(map[discover.NodeID]TestPeer)}
+}
+
+func (self *TestPeerPool) Add(p TestPeer) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	log.Trace(fmt.Sprintf("pp add peer  %v", p.ID()))
+	self.peers[p.ID()] = p
+
+}
+
+func (self *TestPeerPool) Remove(p TestPeer) {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	delete(self.peers, p.ID())
+}
+
+func (self *TestPeerPool) Has(id discover.NodeID) bool {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	_, ok := self.peers[id]
+	return ok
+}
+
+func (self *TestPeerPool) Get(id discover.NodeID) TestPeer {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+	return self.peers[id]
+}

p2p/testing/protocolsession.go

@@ -0,0 +1,280 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package testing
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/EthereumCommonwealth/go-callisto/log"
+	"github.com/EthereumCommonwealth/go-callisto/p2p"
+	"github.com/EthereumCommonwealth/go-callisto/p2p/discover"
+	"github.com/EthereumCommonwealth/go-callisto/p2p/simulations/adapters"
+)
+
+var errTimedOut = errors.New("timed out")
+
+// ProtocolSession is a quasi simulation of a pivot node running
+// a service and a number of dummy peers that can send (trigger) or
+// receive (expect) messages
+type ProtocolSession struct {
+	Server  *p2p.Server
+	IDs     []discover.NodeID
+	adapter *adapters.SimAdapter
+	events  chan *p2p.PeerEvent
+}
+
+// Exchange is the basic units of protocol tests
+// the triggers and expects in the arrays are run immediately and asynchronously
+// thus one cannot have multiple expects for the SAME peer with DIFFERENT message types
+// because it's unpredictable which expect will receive which message
+// (with expect #1 and #2, messages might be sent #2 and #1, and both expects will complain about wrong message code)
+// an exchange is defined on a session
+type Exchange struct {
+	Label    string
+	Triggers []Trigger
+	Expects  []Expect
+	Timeout  time.Duration
+}
+
+// Trigger is part of the exchange, incoming message for the pivot node
+// sent by a peer
+type Trigger struct {
+	Msg     interface{}     // type of message to be sent
+	Code    uint64          // code of message is given
+	Peer    discover.NodeID // the peer to send the message to
+	Timeout time.Duration   // timeout duration for the sending
+}
+
+// Expect is part of an exchange, outgoing message from the pivot node
+// received by a peer
+type Expect struct {
+	Msg     interface{}     // type of message to expect
+	Code    uint64          // code of message is now given
+	Peer    discover.NodeID // the peer that expects the message
+	Timeout time.Duration   // timeout duration for receiving
+}
+
+// Disconnect represents a disconnect event, used and checked by TestDisconnected
+type Disconnect struct {
+	Peer  discover.NodeID // discconnected peer
+	Error error           // disconnect reason
+}
+
+// trigger sends messages from peers
+func (self *ProtocolSession) trigger(trig Trigger) error {
+	simNode, ok := self.adapter.GetNode(trig.Peer)
+	if !ok {
+		return fmt.Errorf("trigger: peer %v does not exist (1- %v)", trig.Peer, len(self.IDs))
+	}
+	mockNode, ok := simNode.Services()[0].(*mockNode)
+	if !ok {
+		return fmt.Errorf("trigger: peer %v is not a mock", trig.Peer)
+	}
+
+	errc := make(chan error)
+
+	go func() {
+		errc <- mockNode.Trigger(&trig)
+	}()
+
+	t := trig.Timeout
+	if t == time.Duration(0) {
+		t = 1000 * time.Millisecond
+	}
+	select {
+	case err := <-errc:
+		return err
+	case <-time.After(t):
+		return fmt.Errorf("timout expecting %v to send to peer %v", trig.Msg, trig.Peer)
+	}
+}
+
+// expect checks an expectation of a message sent out by the pivot node
+func (self *ProtocolSession) expect(exps []Expect) error {
+	// construct a map of expectations for each node
+	peerExpects := make(map[discover.NodeID][]Expect)
+	for _, exp := range exps {
+		if exp.Msg == nil {
+			return errors.New("no message to expect")
+		}
+		peerExpects[exp.Peer] = append(peerExpects[exp.Peer], exp)
+	}
+
+	// construct a map of mockNodes for each node
+	mockNodes := make(map[discover.NodeID]*mockNode)
+	for nodeID := range peerExpects {
+		simNode, ok := self.adapter.GetNode(nodeID)
+		if !ok {
+			return fmt.Errorf("trigger: peer %v does not exist (1- %v)", nodeID, len(self.IDs))
+		}
+		mockNode, ok := simNode.Services()[0].(*mockNode)
+		if !ok {
+			return fmt.Errorf("trigger: peer %v is not a mock", nodeID)
+		}
+		mockNodes[nodeID] = mockNode
+	}
+
+	// done chanell cancels all created goroutines when function returns
+	done := make(chan struct{})
+	defer close(done)
+	// errc catches the first error from
+	errc := make(chan error)
+
+	wg := &sync.WaitGroup{}
+	wg.Add(len(mockNodes))
+	for nodeID, mockNode := range mockNodes {
+		nodeID := nodeID
+		mockNode := mockNode
+		go func() {
+			defer wg.Done()
+
+			// Sum all Expect timeouts to give the maximum
+			// time for all expectations to finish.
+			// mockNode.Expect checks all received messages against
+			// a list of expected messages and timeout for each
+			// of them can not be checked separately.
+			var t time.Duration
+			for _, exp := range peerExpects[nodeID] {
+				if exp.Timeout == time.Duration(0) {
+					t += 2000 * time.Millisecond
+				} else {
+					t += exp.Timeout
+				}
+			}
+			alarm := time.NewTimer(t)
+			defer alarm.Stop()
+
+			// expectErrc is used to check if error returned
+			// from mockNode.Expect is not nil and to send it to
+			// errc only in that case.
+			// done channel will be closed when function
+			expectErrc := make(chan error)
+			go func() {
+				select {
+				case expectErrc <- mockNode.Expect(peerExpects[nodeID]...):
+				case <-done:
+				case <-alarm.C:
+				}
+			}()
+
+			select {
+			case err := <-expectErrc:
+				if err != nil {
+					select {
+					case errc <- err:
+					case <-done:
+					case <-alarm.C:
+						errc <- errTimedOut
+					}
+				}
+			case <-done:
+			case <-alarm.C:
+				errc <- errTimedOut
+			}
+
+		}()
+	}
+
+	go func() {
+		wg.Wait()
+		// close errc when all goroutines finish to return nill err from errc
+		close(errc)
+	}()
+
+	return <-errc
+}
+
+// TestExchanges tests a series of exchanges against the session
+func (self *ProtocolSession) TestExchanges(exchanges ...Exchange) error {
+	for i, e := range exchanges {
+		if err := self.testExchange(e); err != nil {
+			return fmt.Errorf("exchange #%d %q: %v", i, e.Label, err)
+		}
+		log.Trace(fmt.Sprintf("exchange #%d %q: run successfully", i, e.Label))
+	}
+	return nil
+}
+
+// testExchange tests a single Exchange.
+// Default timeout value is 2 seconds.
+func (self *ProtocolSession) testExchange(e Exchange) error {
+	errc := make(chan error)
+	done := make(chan struct{})
+	defer close(done)
+
+	go func() {
+		for _, trig := range e.Triggers {
+			err := self.trigger(trig)
+			if err != nil {
+				errc <- err
+				return
+			}
+		}
+
+		select {
+		case errc <- self.expect(e.Expects):
+		case <-done:
+		}
+	}()
+
+	// time out globally or finish when all expectations satisfied
+	t := e.Timeout
+	if t == 0 {
+		t = 2000 * time.Millisecond
+	}
+	alarm := time.NewTimer(t)
+	select {
+	case err := <-errc:
+		return err
+	case <-alarm.C:
+		return errTimedOut
+	}
+}
+
+// TestDisconnected tests the disconnections given as arguments
+// the disconnect structs describe what disconnect error is expected on which peer
+func (self *ProtocolSession) TestDisconnected(disconnects ...*Disconnect) error {
+	expects := make(map[discover.NodeID]error)
+	for _, disconnect := range disconnects {
+		expects[disconnect.Peer] = disconnect.Error
+	}
+
+	timeout := time.After(time.Second)
+	for len(expects) > 0 {
+		select {
+		case event := <-self.events:
+			if event.Type != p2p.PeerEventTypeDrop {
+				continue
+			}
+			expectErr, ok := expects[event.Peer]
+			if !ok {
+				continue
+			}
+
+			if !(expectErr == nil && event.Error == "" || expectErr != nil && expectErr.Error() == event.Error) {
+				return fmt.Errorf("unexpected error on peer %v. expected '%v', got '%v'", event.Peer, expectErr, event.Error)
+			}
+			delete(expects, event.Peer)
+		case <-timeout:
+			return fmt.Errorf("timed out waiting for peers to disconnect")
+		}
+	}
+	return nil
+}

p2p/testing/protocoltester.go

@@ -0,0 +1,269 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+/*
+the p2p/testing package provides a unit test scheme to check simple
+protocol message exchanges with one pivot node and a number of dummy peers
+The pivot test node runs a node.Service, the dummy peers run a mock node
+that can be used to send and receive messages
+*/
+
+package testing
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/EthereumCommonwealth/go-callisto/log"
+	"github.com/EthereumCommonwealth/go-callisto/node"
+	"github.com/EthereumCommonwealth/go-callisto/p2p"
+	"github.com/EthereumCommonwealth/go-callisto/p2p/discover"
+	"github.com/EthereumCommonwealth/go-callisto/p2p/simulations"
+	"github.com/EthereumCommonwealth/go-callisto/p2p/simulations/adapters"
+	"github.com/EthereumCommonwealth/go-callisto/rlp"
+	"github.com/EthereumCommonwealth/go-callisto/rpc"
+)
+
+// ProtocolTester is the tester environment used for unit testing protocol
+// message exchanges. It uses p2p/simulations framework
+type ProtocolTester struct {
+	*ProtocolSession
+	network *simulations.Network
+}
+
+// NewProtocolTester constructs a new ProtocolTester
+// it takes as argument the pivot node id, the number of dummy peers and the
+// protocol run function called on a peer connection by the p2p server
+func NewProtocolTester(t *testing.T, id discover.NodeID, n int, run func(*p2p.Peer, p2p.MsgReadWriter) error) *ProtocolTester {
+	services := adapters.Services{
+		"test": func(ctx *adapters.ServiceContext) (node.Service, error) {
+			return &testNode{run}, nil
+		},
+		"mock": func(ctx *adapters.ServiceContext) (node.Service, error) {
+			return newMockNode(), nil
+		},
+	}
+	adapter := adapters.NewSimAdapter(services)
+	net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{})
+	if _, err := net.NewNodeWithConfig(&adapters.NodeConfig{
+		ID:              id,
+		EnableMsgEvents: true,
+		Services:        []string{"test"},
+	}); err != nil {
+		panic(err.Error())
+	}
+	if err := net.Start(id); err != nil {
+		panic(err.Error())
+	}
+
+	node := net.GetNode(id).Node.(*adapters.SimNode)
+	peers := make([]*adapters.NodeConfig, n)
+	peerIDs := make([]discover.NodeID, n)
+	for i := 0; i < n; i++ {
+		peers[i] = adapters.RandomNodeConfig()
+		peers[i].Services = []string{"mock"}
+		peerIDs[i] = peers[i].ID
+	}
+	events := make(chan *p2p.PeerEvent, 1000)
+	node.SubscribeEvents(events)
+	ps := &ProtocolSession{
+		Server:  node.Server(),
+		IDs:     peerIDs,
+		adapter: adapter,
+		events:  events,
+	}
+	self := &ProtocolTester{
+		ProtocolSession: ps,
+		network:         net,
+	}
+
+	self.Connect(id, peers...)
+
+	return self
+}
+
+// Stop stops the p2p server
+func (self *ProtocolTester) Stop() error {
+	self.Server.Stop()
+	return nil
+}
+
+// Connect brings up the remote peer node and connects it using the
+// p2p/simulations network connection with the in memory network adapter
+func (self *ProtocolTester) Connect(selfID discover.NodeID, peers ...*adapters.NodeConfig) {
+	for _, peer := range peers {
+		log.Trace(fmt.Sprintf("start node %v", peer.ID))
+		if _, err := self.network.NewNodeWithConfig(peer); err != nil {
+			panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err))
+		}
+		if err := self.network.Start(peer.ID); err != nil {
+			panic(fmt.Sprintf("error starting peer %v: %v", peer.ID, err))
+		}
+		log.Trace(fmt.Sprintf("connect to %v", peer.ID))
+		if err := self.network.Connect(selfID, peer.ID); err != nil {
+			panic(fmt.Sprintf("error connecting to peer %v: %v", peer.ID, err))
+		}
+	}
+
+}
+
+// testNode wraps a protocol run function and implements the node.Service
+// interface
+type testNode struct {
+	run func(*p2p.Peer, p2p.MsgReadWriter) error
+}
+
+func (t *testNode) Protocols() []p2p.Protocol {
+	return []p2p.Protocol{{
+		Length: 100,
+		Run:    t.run,
+	}}
+}
+
+func (t *testNode) APIs() []rpc.API {
+	return nil
+}
+
+func (t *testNode) Start(server *p2p.Server) error {
+	return nil
+}
+
+func (t *testNode) Stop() error {
+	return nil
+}
+
+// mockNode is a testNode which doesn't actually run a protocol, instead
+// exposing channels so that tests can manually trigger and expect certain
+// messages
+type mockNode struct {
+	testNode
+
+	trigger  chan *Trigger
+	expect   chan []Expect
+	err      chan error
+	stop     chan struct{}
+	stopOnce sync.Once
+}
+
+func newMockNode() *mockNode {
+	mock := &mockNode{
+		trigger: make(chan *Trigger),
+		expect:  make(chan []Expect),
+		err:     make(chan error),
+		stop:    make(chan struct{}),
+	}
+	mock.testNode.run = mock.Run
+	return mock
+}
+
+// Run is a protocol run function which just loops waiting for tests to
+// instruct it to either trigger or expect a message from the peer
+func (m *mockNode) Run(peer *p2p.Peer, rw p2p.MsgReadWriter) error {
+	for {
+		select {
+		case trig := <-m.trigger:
+			m.err <- p2p.Send(rw, trig.Code, trig.Msg)
+		case exps := <-m.expect:
+			m.err <- expectMsgs(rw, exps)
+		case <-m.stop:
+			return nil
+		}
+	}
+}
+
+func (m *mockNode) Trigger(trig *Trigger) error {
+	m.trigger <- trig
+	return <-m.err
+}
+
+func (m *mockNode) Expect(exp ...Expect) error {
+	m.expect <- exp
+	return <-m.err
+}
+
+func (m *mockNode) Stop() error {
+	m.stopOnce.Do(func() { close(m.stop) })
+	return nil
+}
+
+func expectMsgs(rw p2p.MsgReadWriter, exps []Expect) error {
+	matched := make([]bool, len(exps))
+	for {
+		msg, err := rw.ReadMsg()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return err
+		}
+		actualContent, err := ioutil.ReadAll(msg.Payload)
+		if err != nil {
+			return err
+		}
+		var found bool
+		for i, exp := range exps {
+			if exp.Code == msg.Code && bytes.Equal(actualContent, mustEncodeMsg(exp.Msg)) {
+				if matched[i] {
+					return fmt.Errorf("message #%d received two times", i)
+				}
+				matched[i] = true
+				found = true
+				break
+			}
+		}
+		if !found {
+			expected := make([]string, 0)
+			for i, exp := range exps {
+				if matched[i] {
+					continue
+				}
+				expected = append(expected, fmt.Sprintf("code %d payload %x", exp.Code, mustEncodeMsg(exp.Msg)))
+			}
+			return fmt.Errorf("unexpected message code %d payload %x, expected %s", msg.Code, actualContent, strings.Join(expected, " or "))
+		}
+		done := true
+		for _, m := range matched {
+			if !m {
+				done = false
+				break
+			}
+		}
+		if done {
+			return nil
+		}
+	}
+	for i, m := range matched {
+		if !m {
+			return fmt.Errorf("expected message #%d not received", i)
+		}
+	}
+	return nil
+}
+
+// mustEncodeMsg uses rlp to encode a message.
+// In case of error it panics.
+func mustEncodeMsg(msg interface{}) []byte {
+	contentEnc, err := rlp.EncodeToBytes(msg)
+	if err != nil {
+		panic("content encode error: " + err.Error())
+	}
+	return contentEnc
+}