From f0998415ba9a73f0add32f9b5aed2aec98b9a7f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 8 Aug 2018 12:15:08 +0300 Subject: [PATCH 01/96] cmd, consensus/ethash, eth: miner push notifications --- cmd/geth/main.go | 3 +- cmd/geth/usage.go | 1 + cmd/utils/flags.go | 14 +++- consensus/ethash/algorithm_test.go | 2 +- consensus/ethash/consensus.go | 2 +- consensus/ethash/ethash.go | 18 +++-- consensus/ethash/ethash_test.go | 43 +++++------ consensus/ethash/sealer.go | 90 ++++++++++++++-------- consensus/ethash/sealer_test.go | 115 +++++++++++++++++++++++++++++ eth/backend.go | 8 +- eth/config.go | 1 + les/backend.go | 2 +- 12 files changed, 226 insertions(+), 73 deletions(-) create mode 100644 consensus/ethash/sealer_test.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 77ef6afe24..d556ad92c3 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -98,8 +98,9 @@ var ( utils.MaxPendingPeersFlag, utils.EtherbaseFlag, utils.GasPriceFlag, - utils.MinerThreadsFlag, utils.MiningEnabledFlag, + utils.MinerThreadsFlag, + utils.MinerNotifyFlag, utils.TargetGasLimitFlag, utils.NATFlag, utils.NoDiscoverFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 6a12a66cc2..9d63c68f7f 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -185,6 +185,7 @@ var AppHelpFlagGroups = []flagGroup{ Flags: []cli.Flag{ utils.MiningEnabledFlag, utils.MinerThreadsFlag, + utils.MinerNotifyFlag, utils.EtherbaseFlag, utils.TargetGasLimitFlag, utils.GasPriceFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 522ad06b61..d6142f246c 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -24,7 +24,6 @@ import ( "math/big" "os" "path/filepath" - "runtime" "strconv" "strings" "time" @@ -318,9 +317,13 @@ var ( Usage: "Enable mining", } MinerThreadsFlag = cli.IntFlag{ - Name: "minerthreads", + Name: "miner.threads", Usage: "Number of CPU threads to use for mining", - Value: runtime.NumCPU(), + Value: 0, + } + MinerNotifyFlag = cli.StringFlag{ + Name: "miner.notify", + Usage: "Comma separated HTTP URL list to notify of new work packages", } TargetGasLimitFlag = cli.Uint64Flag{ Name: "targetgaslimit", @@ -1093,6 +1096,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(MinerThreadsFlag.Name) { cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name) } + if ctx.GlobalIsSet(MinerNotifyFlag.Name) { + cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") + } if ctx.GlobalIsSet(DocRootFlag.Name) { cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) } @@ -1293,7 +1299,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir), DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem, DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk, - }) + }, nil) } } if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go index e7625f7c00..db22cccd0c 100644 --- a/consensus/ethash/algorithm_test.go +++ b/consensus/ethash/algorithm_test.go @@ -729,7 +729,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) { go func(idx int) { defer pend.Done() - ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}) + ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}, nil) defer ethash.Close() if err := ethash.VerifySeal(nil, block.Header()); err != nil { t.Errorf("proc %d: block verification failed: %v", idx, err) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index eb0f73d98b..e18a06d52a 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -493,7 +493,7 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head if !bytes.Equal(header.MixDigest[:], digest) { return errInvalidMixDigest } - target := new(big.Int).Div(maxUint256, header.Difficulty) + target := new(big.Int).Div(two256, header.Difficulty) if new(big.Int).SetBytes(result).Cmp(target) > 0 { return errInvalidPoW } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 0cb3059b9d..19c94deb6b 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -45,11 +45,11 @@ import ( var ErrInvalidDumpMagic = errors.New("invalid dump magic") var ( - // maxUint256 is a big integer representing 2^256-1 - maxUint256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) + // two256 is a big integer representing 2^256 + two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) // sharedEthash is a full instance that can be shared between multiple users. - sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}) + sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}, nil) // algorithmRevision is the data structure version used for file naming. algorithmRevision = 23 @@ -447,8 +447,10 @@ type Ethash struct { exitCh chan chan error // Notification channel to exiting backend threads } -// New creates a full sized ethash PoW scheme and starts a background thread for remote mining. -func New(config Config) *Ethash { +// New creates a full sized ethash PoW scheme and starts a background thread for +// remote mining, also optionally notifying a batch of remote services of new work +// packages. +func New(config Config, notify []string) *Ethash { if config.CachesInMem <= 0 { log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem) config.CachesInMem = 1 @@ -473,13 +475,13 @@ func New(config Config) *Ethash { submitRateCh: make(chan *hashrate), exitCh: make(chan chan error), } - go ethash.remote() + go ethash.remote(notify) return ethash } // NewTester creates a small sized ethash PoW scheme useful only for testing // purposes. -func NewTester() *Ethash { +func NewTester(notify []string) *Ethash { ethash := &Ethash{ config: Config{PowMode: ModeTest}, caches: newlru("cache", 1, newCache), @@ -494,7 +496,7 @@ func NewTester() *Ethash { submitRateCh: make(chan *hashrate), exitCh: make(chan chan error), } - go ethash.remote() + go ethash.remote(notify) return ethash } diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index ccdd30fb0f..87ac17c2b2 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -32,17 +32,18 @@ import ( // Tests that ethash works correctly in test mode. func TestTestMode(t *testing.T) { - head := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} + header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} - ethash := NewTester() + ethash := NewTester(nil) defer ethash.Close() - block, err := ethash.Seal(nil, types.NewBlockWithHeader(head), nil) + + block, err := ethash.Seal(nil, types.NewBlockWithHeader(header), nil) if err != nil { t.Fatalf("failed to seal block: %v", err) } - head.Nonce = types.EncodeNonce(block.Nonce()) - head.MixDigest = block.MixDigest() - if err := ethash.VerifySeal(nil, head); err != nil { + header.Nonce = types.EncodeNonce(block.Nonce()) + header.MixDigest = block.MixDigest() + if err := ethash.VerifySeal(nil, header); err != nil { t.Fatalf("unexpected verification error: %v", err) } } @@ -55,7 +56,7 @@ func TestCacheFileEvict(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmpdir) - e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}) + e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil) defer e.Close() workers := 8 @@ -78,21 +79,21 @@ func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) { if block < 0 { block = 0 } - head := &types.Header{Number: big.NewInt(block), Difficulty: big.NewInt(100)} - e.VerifySeal(nil, head) + header := &types.Header{Number: big.NewInt(block), Difficulty: big.NewInt(100)} + e.VerifySeal(nil, header) } } func TestRemoteSealer(t *testing.T) { - ethash := NewTester() + ethash := NewTester(nil) defer ethash.Close() + api := &API{ethash} if _, err := api.GetWork(); err != errNoMiningWork { t.Error("expect to return an error indicate there is no mining work") } - - head := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} - block := types.NewBlockWithHeader(head) + header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} + block := types.NewBlockWithHeader(header) // Push new work. ethash.Seal(nil, block, nil) @@ -108,16 +109,14 @@ func TestRemoteSealer(t *testing.T) { if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res { t.Error("expect to return false when submit a fake solution") } - // Push new block with same block number to replace the original one. - head = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)} - block = types.NewBlockWithHeader(head) + header = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)} + block = types.NewBlockWithHeader(header) ethash.Seal(nil, block, nil) if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() { t.Error("expect to return the latest pushed work") } - // Push block with higher block number. newHead := &types.Header{Number: big.NewInt(2), Difficulty: big.NewInt(100)} newBlock := types.NewBlockWithHeader(newHead) @@ -130,19 +129,18 @@ func TestRemoteSealer(t *testing.T) { func TestHashRate(t *testing.T) { var ( - ethash = NewTester() - api = &API{ethash} hashrate = []hexutil.Uint64{100, 200, 300} expect uint64 ids = []common.Hash{common.HexToHash("a"), common.HexToHash("b"), common.HexToHash("c")} ) - + ethash := NewTester(nil) defer ethash.Close() if tot := ethash.Hashrate(); tot != 0 { t.Error("expect the result should be zero") } + api := &API{ethash} for i := 0; i < len(hashrate); i += 1 { if res := api.SubmitHashRate(hashrate[i], ids[i]); !res { t.Error("remote miner submit hashrate failed") @@ -155,9 +153,8 @@ func TestHashRate(t *testing.T) { } func TestClosedRemoteSealer(t *testing.T) { - ethash := NewTester() - // Make sure exit channel has been listened - time.Sleep(1 * time.Second) + ethash := NewTester(nil) + time.Sleep(1 * time.Second) // ensure exit channel is listening ethash.Close() api := &API{ethash} diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index a9449d4060..03d8484739 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -17,11 +17,14 @@ package ethash import ( + "bytes" crand "crypto/rand" + "encoding/json" "errors" "math" "math/big" "math/rand" + "net/http" "runtime" "sync" "time" @@ -109,7 +112,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s var ( header = block.Header() hash = header.HashNoNonce().Bytes() - target = new(big.Int).Div(maxUint256, header.Difficulty) + target = new(big.Int).Div(two256, header.Difficulty) number = header.Number.Uint64() dataset = ethash.dataset(number) ) @@ -161,40 +164,65 @@ search: runtime.KeepAlive(dataset) } -// remote starts a standalone goroutine to handle remote mining related stuff. -func (ethash *Ethash) remote() { +// remote is a standalone goroutine to handle remote mining related stuff. +func (ethash *Ethash) remote(notify []string) { var ( - works = make(map[common.Hash]*types.Block) - rates = make(map[common.Hash]hashrate) - currentWork *types.Block - ) + works = make(map[common.Hash]*types.Block) + rates = make(map[common.Hash]hashrate) - // getWork returns a work package for external miner. + currentBlock *types.Block + currentWork [3]string + + notifyTransport = &http.Transport{} + notifyClient = &http.Client{ + Transport: notifyTransport, + Timeout: time.Second, + } + notifyReqs = make([]*http.Request, len(notify)) + ) + // notifyWork notifies all the specified mining endpoints of the availability of + // new work to be processed. + notifyWork := func() { + work := currentWork + blob, _ := json.Marshal(work) + + for i, url := range notify { + // Terminate any previously pending request and create the new work + if notifyReqs[i] != nil { + notifyTransport.CancelRequest(notifyReqs[i]) + } + notifyReqs[i], _ = http.NewRequest("POST", url, bytes.NewReader(blob)) + notifyReqs[i].Header.Set("Content-Type", "application/json") + + // Push the new work concurrently to all the remote nodes + go func(req *http.Request, url string) { + res, err := notifyClient.Do(req) + if err != nil { + log.Warn("Failed to notify remote miner", "err", err) + } else { + log.Trace("Notified remote miner", "miner", url, "hash", log.Lazy{Fn: func() common.Hash { return common.HexToHash(work[0]) }}, "target", work[2]) + res.Body.Close() + } + }(notifyReqs[i], url) + } + } + // makeWork creates a work package for external miner. // // The work package consists of 3 strings: // result[0], 32 bytes hex encoded current block header pow-hash // result[1], 32 bytes hex encoded seed hash used for DAG // result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty - getWork := func() ([3]string, error) { - var res [3]string - if currentWork == nil { - return res, errNoMiningWork - } - res[0] = currentWork.HashNoNonce().Hex() - res[1] = common.BytesToHash(SeedHash(currentWork.NumberU64())).Hex() + makeWork := func(block *types.Block) { + hash := block.HashNoNonce() - // Calculate the "target" to be returned to the external sealer. - n := big.NewInt(1) - n.Lsh(n, 255) - n.Div(n, currentWork.Difficulty()) - n.Lsh(n, 1) - res[2] = common.BytesToHash(n.Bytes()).Hex() + currentWork[0] = hash.Hex() + currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex() + currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex() // Trace the seal work fetched by remote sealer. - works[currentWork.HashNoNonce()] = currentWork - return res, nil + currentBlock = block + works[hash] = block } - // submitWork verifies the submitted pow solution, returning // whether the solution was accepted or not (not can be both a bad pow as well as // any other error, like no pending work or stale mining result). @@ -238,21 +266,23 @@ func (ethash *Ethash) remote() { for { select { case block := <-ethash.workCh: - if currentWork != nil && block.ParentHash() != currentWork.ParentHash() { + if currentBlock != nil && block.ParentHash() != currentBlock.ParentHash() { // Start new round mining, throw out all previous work. works = make(map[common.Hash]*types.Block) } // Update current work with new received block. // Note same work can be past twice, happens when changing CPU threads. - currentWork = block + makeWork(block) + + // Notify and requested URLs of the new work availability + notifyWork() case work := <-ethash.fetchWorkCh: // Return current mining work to remote miner. - miningWork, err := getWork() - if err != nil { - work.errc <- err + if currentBlock == nil { + work.errc <- errNoMiningWork } else { - work.res <- miningWork + work.res <- currentWork } case result := <-ethash.submitWorkCh: diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go new file mode 100644 index 0000000000..6d8a770495 --- /dev/null +++ b/consensus/ethash/sealer_test.go @@ -0,0 +1,115 @@ +package ethash + +import ( + "encoding/json" + "io/ioutil" + "math/big" + "net" + "net/http" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// Tests whether remote HTTP servers are correctly notified of new work. +func TestRemoteNotify(t *testing.T) { + // Start a simple webserver to capture notifications + sink := make(chan [3]string) + + server := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + blob, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Fatalf("failed to read miner notification: %v", err) + } + var work [3]string + if err := json.Unmarshal(blob, &work); err != nil { + t.Fatalf("failed to unmarshal miner notification: %v", err) + } + sink <- work + }), + } + // Open a custom listener to extract its local address + listener, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to open notification server: %v", err) + } + defer listener.Close() + + go server.Serve(listener) + + // Create the custom ethash engine + ethash := NewTester([]string{"http://" + listener.Addr().String()}) + defer ethash.Close() + + // Stream a work task and ensure the notification bubbles out + header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} + block := types.NewBlockWithHeader(header) + + ethash.Seal(nil, block, nil) + select { + case work := <-sink: + if want := header.HashNoNonce().Hex(); work[0] != want { + t.Errorf("work packet hash mismatch: have %s, want %s", work[0], want) + } + if want := common.BytesToHash(SeedHash(header.Number.Uint64())).Hex(); work[1] != want { + t.Errorf("work packet seed mismatch: have %s, want %s", work[1], want) + } + target := new(big.Int).Div(new(big.Int).Lsh(big.NewInt(1), 256), header.Difficulty) + if want := common.BytesToHash(target.Bytes()).Hex(); work[2] != want { + t.Errorf("work packet target mismatch: have %s, want %s", work[2], want) + } + case <-time.After(time.Second): + t.Fatalf("notification timed out") + } +} + +// Tests that pushing work packages fast to the miner doesn't cause any daa race +// issues in the notifications. +func TestRemoteMultiNotify(t *testing.T) { + // Start a simple webserver to capture notifications + sink := make(chan [3]string, 1024) + + server := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + blob, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Fatalf("failed to read miner notification: %v", err) + } + var work [3]string + if err := json.Unmarshal(blob, &work); err != nil { + t.Fatalf("failed to unmarshal miner notification: %v", err) + } + sink <- work + }), + } + // Open a custom listener to extract its local address + listener, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("failed to open notification server: %v", err) + } + defer listener.Close() + + go server.Serve(listener) + + // Create the custom ethash engine + ethash := NewTester([]string{"http://" + listener.Addr().String()}) + defer ethash.Close() + + // Stream a lot of work task and ensure all the notifications bubble out + for i := 0; i < cap(sink); i++ { + header := &types.Header{Number: big.NewInt(int64(i)), Difficulty: big.NewInt(100)} + block := types.NewBlockWithHeader(header) + + ethash.Seal(nil, block, nil) + } + for i := 0; i < cap(sink); i++ { + select { + case <-sink: + case <-time.After(250 * time.Millisecond): + t.Fatalf("notification %d timed out", i) + } + } +} diff --git a/eth/backend.go b/eth/backend.go index 32946a0ab3..865534b198 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -124,7 +124,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { chainConfig: chainConfig, eventMux: ctx.EventMux, accountManager: ctx.AccountManager, - engine: CreateConsensusEngine(ctx, &config.Ethash, chainConfig, chainDb), + engine: CreateConsensusEngine(ctx, chainConfig, &config.Ethash, config.MinerNotify, chainDb), shutdownChan: make(chan bool), networkID: config.NetworkId, gasPrice: config.GasPrice, @@ -210,7 +210,7 @@ func CreateDB(ctx *node.ServiceContext, config *Config, name string) (ethdb.Data } // CreateConsensusEngine creates the required type of consensus engine instance for an Ethereum service -func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chainConfig *params.ChainConfig, db ethdb.Database) consensus.Engine { +func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, db ethdb.Database) consensus.Engine { // If proof-of-authority is requested, set it up if chainConfig.Clique != nil { return clique.New(chainConfig.Clique, db) @@ -222,7 +222,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai return ethash.NewFaker() case ethash.ModeTest: log.Warn("Ethash used in test mode") - return ethash.NewTester() + return ethash.NewTester(nil) case ethash.ModeShared: log.Warn("Ethash used in shared mode") return ethash.NewShared() @@ -234,7 +234,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, config *ethash.Config, chai DatasetDir: config.DatasetDir, DatasetsInMem: config.DatasetsInMem, DatasetsOnDisk: config.DatasetsOnDisk, - }) + }, notify) engine.SetThreads(-1) // Disable CPU mining return engine } diff --git a/eth/config.go b/eth/config.go index 426d2bf1ef..0c82f29232 100644 --- a/eth/config.go +++ b/eth/config.go @@ -97,6 +97,7 @@ type Config struct { // Mining-related options Etherbase common.Address `toml:",omitempty"` MinerThreads int `toml:",omitempty"` + MinerNotify []string `toml:",omitempty"` ExtraData []byte `toml:",omitempty"` GasPrice *big.Int diff --git a/les/backend.go b/les/backend.go index 952d92cc2a..178bc1e0e4 100644 --- a/les/backend.go +++ b/les/backend.go @@ -102,7 +102,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { peers: peers, reqDist: newRequestDistributor(peers, quitSync), accountManager: ctx.AccountManager, - engine: eth.CreateConsensusEngine(ctx, &config.Ethash, chainConfig, chainDb), + engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, chainDb), shutdownChan: make(chan bool), networkId: config.NetworkId, bloomRequests: make(chan chan *bloombits.Retrieval), From e07e507d1af687cd64b263038ebd3cb7be74fb40 Mon Sep 17 00:00:00 2001 From: Eugene Valeyev Date: Mon, 13 Aug 2018 17:27:25 +0300 Subject: [PATCH 02/96] whisper: fixed broken partial topic filtering Changes in #15811 broke partial topic filtering. Re-enable it. --- whisper/whisperv5/filter.go | 2 +- whisper/whisperv5/filter_test.go | 8 +++---- whisper/whisperv6/filter.go | 17 --------------- whisper/whisperv6/filter_test.go | 36 -------------------------------- 4 files changed, 5 insertions(+), 58 deletions(-) diff --git a/whisper/whisperv5/filter.go b/whisper/whisperv5/filter.go index 3190334ebb..9550a7e389 100644 --- a/whisper/whisperv5/filter.go +++ b/whisper/whisperv5/filter.go @@ -220,7 +220,7 @@ func matchSingleTopic(topic TopicType, bt []byte) bool { bt = bt[:TopicLength] } - if len(bt) < TopicLength { + if len(bt) == 0 { return false } diff --git a/whisper/whisperv5/filter_test.go b/whisper/whisperv5/filter_test.go index 01034a3513..c01c22668c 100644 --- a/whisper/whisperv5/filter_test.go +++ b/whisper/whisperv5/filter_test.go @@ -829,16 +829,16 @@ func TestMatchSingleTopic_WithTail_ReturnTrue(t *testing.T) { } } -func TestMatchSingleTopic_NotEquals_ReturnFalse(t *testing.T) { +func TestMatchSingleTopic_PartialTopic_ReturnTrue(t *testing.T) { bt := []byte("tes") - topic := BytesToTopic(bt) + topic := BytesToTopic([]byte("test")) - if matchSingleTopic(topic, bt) { + if !matchSingleTopic(topic, bt) { t.FailNow() } } -func TestMatchSingleTopic_InsufficientLength_ReturnFalse(t *testing.T) { +func TestMatchSingleTopic_NotEquals_ReturnFalse(t *testing.T) { bt := []byte("test") topic := BytesToTopic([]byte("not_equal")) diff --git a/whisper/whisperv6/filter.go b/whisper/whisperv6/filter.go index 2f170ddebe..6a5b79674b 100644 --- a/whisper/whisperv6/filter.go +++ b/whisper/whisperv6/filter.go @@ -250,23 +250,6 @@ func (f *Filter) MatchEnvelope(envelope *Envelope) bool { return f.PoW <= 0 || envelope.pow >= f.PoW } -func matchSingleTopic(topic TopicType, bt []byte) bool { - if len(bt) > TopicLength { - bt = bt[:TopicLength] - } - - if len(bt) < TopicLength { - return false - } - - for j, b := range bt { - if topic[j] != b { - return false - } - } - return true -} - // IsPubKeyEqual checks that two public keys are equal func IsPubKeyEqual(a, b *ecdsa.PublicKey) bool { if !ValidatePublicKey(a) { diff --git a/whisper/whisperv6/filter_test.go b/whisper/whisperv6/filter_test.go index 0bb7986c39..82e4aa0241 100644 --- a/whisper/whisperv6/filter_test.go +++ b/whisper/whisperv6/filter_test.go @@ -829,39 +829,3 @@ func TestVariableTopics(t *testing.T) { } } } - -func TestMatchSingleTopic_ReturnTrue(t *testing.T) { - bt := []byte("test") - topic := BytesToTopic(bt) - - if !matchSingleTopic(topic, bt) { - t.FailNow() - } -} - -func TestMatchSingleTopic_WithTail_ReturnTrue(t *testing.T) { - bt := []byte("test with tail") - topic := BytesToTopic([]byte("test")) - - if !matchSingleTopic(topic, bt) { - t.FailNow() - } -} - -func TestMatchSingleTopic_NotEquals_ReturnFalse(t *testing.T) { - bt := []byte("tes") - topic := BytesToTopic(bt) - - if matchSingleTopic(topic, bt) { - t.FailNow() - } -} - -func TestMatchSingleTopic_InsufficientLength_ReturnFalse(t *testing.T) { - bt := []byte("test") - topic := BytesToTopic([]byte("not_equal")) - - if matchSingleTopic(topic, bt) { - t.FailNow() - } -} From 8a040de60bd6b740ebe87cd8e1fe6bfdb6635d2f Mon Sep 17 00:00:00 2001 From: Yao Zengzeng Date: Tue, 14 Aug 2018 19:25:36 +0800 Subject: [PATCH 03/96] README.md: fix some typos (#17381) Signed-off-by: YaoZengzeng --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 86fcdeee4d..c6bc91af17 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ This command will: * Start up Geth's built-in interactive [JavaScript console](https://github.com/ethereum/go-ethereum/wiki/JavaScript-Console), (via the trailing `console` subcommand) through which you can invoke all official [`web3` methods](https://github.com/ethereum/wiki/wiki/JavaScript-API) as well as Geth's own [management APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs). - This too is optional and if you leave it out you can always attach to an already running Geth instance + This tool is optional and if you leave it out you can always attach to an already running Geth instance with `geth attach`. ### Full node on the Ethereum test network From 97887d98da703a31040bceee13bce9ee77fca673 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 14 Aug 2018 16:03:56 +0200 Subject: [PATCH 04/96] swarm/network, swarm/storage: validate chunk size (#17397) * swarm/network, swarm/storage: validate default chunk size * swarm/bmt, swarm/network, swarm/storage: update BMT hash initialisation * swarm/bmt: move segmentCount to tests * swarm/chunk: change chunk.DefaultSize to be untyped const * swarm/storage: add size validator * swarm/storage: add chunk size validation to localstore * swarm/storage: move validation from localstore to validator * swarm/storage: global chunk rules in MRU --- swarm/bmt/bmt.go | 5 +---- swarm/bmt/bmt_test.go | 19 +++++++++++++------ swarm/chunk/chunk.go | 5 +++++ swarm/network/stream/delivery.go | 7 +++++++ swarm/storage/chunker.go | 9 +++------ swarm/storage/hasherstore.go | 9 +++++---- swarm/storage/ldbstore_test.go | 11 ++++++----- swarm/storage/localstore.go | 10 +++------- swarm/storage/localstore_test.go | 8 +++++--- swarm/storage/mru/handler.go | 25 ++++++++----------------- swarm/storage/mru/resource_test.go | 8 +++----- swarm/storage/mru/testutil.go | 5 +---- swarm/storage/mru/update.go | 3 ++- swarm/storage/pyramid.go | 5 +++-- swarm/storage/types.go | 13 ++++++++++--- swarm/swarm.go | 13 ++++--------- 16 files changed, 79 insertions(+), 76 deletions(-) create mode 100644 swarm/chunk/chunk.go diff --git a/swarm/bmt/bmt.go b/swarm/bmt/bmt.go index 97e0e141ed..a85d4369e5 100644 --- a/swarm/bmt/bmt.go +++ b/swarm/bmt/bmt.go @@ -55,9 +55,6 @@ Two implementations are provided: */ const ( - // SegmentCount is the maximum number of segments of the underlying chunk - // Should be equal to max-chunk-data-size / hash-size - SegmentCount = 128 // PoolSize is the maximum number of bmt trees used by the hashers, i.e, // the maximum number of concurrent BMT hashing operations performed by the same hasher PoolSize = 8 @@ -318,7 +315,7 @@ func (h *Hasher) Sum(b []byte) (s []byte) { // with every full segment calls writeSection in a go routine func (h *Hasher) Write(b []byte) (int, error) { l := len(b) - if l == 0 || l > 4096 { + if l == 0 || l > h.pool.Size { return 0, nil } t := h.getTree() diff --git a/swarm/bmt/bmt_test.go b/swarm/bmt/bmt_test.go index 891d8cbb29..760aa11d8b 100644 --- a/swarm/bmt/bmt_test.go +++ b/swarm/bmt/bmt_test.go @@ -34,6 +34,13 @@ import ( // the actual data length generated (could be longer than max datalength of the BMT) const BufferSize = 4128 +const ( + // segmentCount is the maximum number of segments of the underlying chunk + // Should be equal to max-chunk-data-size / hash-size + // Currently set to 128 == 4096 (default chunk size) / 32 (sha3.keccak256 size) + segmentCount = 128 +) + var counts = []int{1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 32, 37, 42, 53, 63, 64, 65, 111, 127, 128} // calculates the Keccak256 SHA3 hash of the data @@ -224,14 +231,14 @@ func TestHasherReuse(t *testing.T) { // tests if bmt reuse is not corrupting result func testHasherReuse(poolsize int, t *testing.T) { hasher := sha3.NewKeccak256 - pool := NewTreePool(hasher, SegmentCount, poolsize) + pool := NewTreePool(hasher, segmentCount, poolsize) defer pool.Drain(0) bmt := New(pool) for i := 0; i < 100; i++ { data := newData(BufferSize) n := rand.Intn(bmt.Size()) - err := testHasherCorrectness(bmt, hasher, data, n, SegmentCount) + err := testHasherCorrectness(bmt, hasher, data, n, segmentCount) if err != nil { t.Fatal(err) } @@ -241,7 +248,7 @@ func testHasherReuse(poolsize int, t *testing.T) { // Tests if pool can be cleanly reused even in concurrent use by several hasher func TestBMTConcurrentUse(t *testing.T) { hasher := sha3.NewKeccak256 - pool := NewTreePool(hasher, SegmentCount, PoolSize) + pool := NewTreePool(hasher, segmentCount, PoolSize) defer pool.Drain(0) cycles := 100 errc := make(chan error) @@ -451,7 +458,7 @@ func benchmarkBMTBaseline(t *testing.B, n int) { func benchmarkBMT(t *testing.B, n int) { data := newData(n) hasher := sha3.NewKeccak256 - pool := NewTreePool(hasher, SegmentCount, PoolSize) + pool := NewTreePool(hasher, segmentCount, PoolSize) bmt := New(pool) t.ReportAllocs() @@ -465,7 +472,7 @@ func benchmarkBMT(t *testing.B, n int) { func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) { data := newData(n) hasher := sha3.NewKeccak256 - pool := NewTreePool(hasher, SegmentCount, PoolSize) + pool := NewTreePool(hasher, segmentCount, PoolSize) bmt := New(pool).NewAsyncWriter(double) idxs, segments := splitAndShuffle(bmt.SectionSize(), data) shuffle(len(idxs), func(i int, j int) { @@ -483,7 +490,7 @@ func benchmarkBMTAsync(t *testing.B, n int, wh whenHash, double bool) { func benchmarkPool(t *testing.B, poolsize, n int) { data := newData(n) hasher := sha3.NewKeccak256 - pool := NewTreePool(hasher, SegmentCount, poolsize) + pool := NewTreePool(hasher, segmentCount, poolsize) cycles := 100 t.ReportAllocs() diff --git a/swarm/chunk/chunk.go b/swarm/chunk/chunk.go new file mode 100644 index 0000000000..1449efccd0 --- /dev/null +++ b/swarm/chunk/chunk.go @@ -0,0 +1,5 @@ +package chunk + +const ( + DefaultSize = 4096 +) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index fa210e3007..36040339d3 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p/discover" + cp "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/spancontext" @@ -229,6 +230,11 @@ R: for req := range d.receiveC { processReceivedChunksCount.Inc(1) + if len(req.SData) > cp.DefaultSize+8 { + log.Warn("received chunk is bigger than expected", "len", len(req.SData)) + continue R + } + // this should be has locally chunk, err := d.db.Get(context.TODO(), req.Addr) if err == nil { @@ -244,6 +250,7 @@ R: continue R default: } + chunk.SData = req.SData d.db.Put(context.TODO(), chunk) diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index b9b5022734..6d805b8e29 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -25,6 +25,7 @@ import ( "time" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/spancontext" opentracing "github.com/opentracing/opentracing-go" @@ -69,10 +70,6 @@ var ( errOperationTimedOut = errors.New("operation timed out") ) -const ( - DefaultChunkSize int64 = 4096 -) - type ChunkerParams struct { chunkSize int64 hashSize int64 @@ -136,7 +133,7 @@ type TreeChunker struct { func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader { jp := &JoinerParams{ ChunkerParams: ChunkerParams{ - chunkSize: DefaultChunkSize, + chunkSize: chunk.DefaultSize, hashSize: int64(len(addr)), }, addr: addr, @@ -156,7 +153,7 @@ func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) ( tsp := &TreeSplitterParams{ SplitterParams: SplitterParams{ ChunkerParams: ChunkerParams{ - chunkSize: DefaultChunkSize, + chunkSize: chunk.DefaultSize, hashSize: putter.RefSize(), }, reader: data, diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 139c0ee031..bc23077c18 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -22,6 +22,7 @@ import ( "sync" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage/encryption" ) @@ -57,7 +58,7 @@ func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool) refSize := int64(hashSize) if toEncrypt { refSize += encryption.KeyLength - chunkEncryption = newChunkEncryption(DefaultChunkSize, refSize) + chunkEncryption = newChunkEncryption(chunk.DefaultSize, refSize) } return &hasherStore{ @@ -190,9 +191,9 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp // removing extra bytes which were just added for padding length := ChunkData(decryptedSpan).Size() - for length > DefaultChunkSize { - length = length + (DefaultChunkSize - 1) - length = length / DefaultChunkSize + for length > chunk.DefaultSize { + length = length + (chunk.DefaultSize - 1) + length = length / chunk.DefaultSize length *= h.refSize } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index baf9e8c142..5ee88baa51 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" @@ -184,7 +185,7 @@ func testIterator(t *testing.T, mock bool) { t.Fatalf("init dbStore failed: %v", err) } - chunks := GenerateRandomChunks(DefaultChunkSize, chunkcount) + chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount) wg := &sync.WaitGroup{} wg.Add(len(chunks)) @@ -294,7 +295,7 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) { chunks := []*Chunk{} for i := 0; i < n; i++ { - c := GenerateRandomChunk(DefaultChunkSize) + c := GenerateRandomChunk(chunk.DefaultSize) chunks = append(chunks, c) log.Trace("generate random chunk", "idx", i, "chunk", c) } @@ -344,7 +345,7 @@ func TestLDBStoreCollectGarbage(t *testing.T) { chunks := []*Chunk{} for i := 0; i < n; i++ { - c := GenerateRandomChunk(DefaultChunkSize) + c := GenerateRandomChunk(chunk.DefaultSize) chunks = append(chunks, c) log.Trace("generate random chunk", "idx", i, "chunk", c) } @@ -398,7 +399,7 @@ func TestLDBStoreAddRemove(t *testing.T) { chunks := []*Chunk{} for i := 0; i < n; i++ { - c := GenerateRandomChunk(DefaultChunkSize) + c := GenerateRandomChunk(chunk.DefaultSize) chunks = append(chunks, c) log.Trace("generate random chunk", "idx", i, "chunk", c) } @@ -460,7 +461,7 @@ func TestLDBStoreRemoveThenCollectGarbage(t *testing.T) { chunks := []*Chunk{} for i := 0; i < capacity; i++ { - c := GenerateRandomChunk(DefaultChunkSize) + c := GenerateRandomChunk(chunk.DefaultSize) chunks = append(chunks, c) log.Trace("generate random chunk", "idx", i, "chunk", c) } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 096d150ae3..9e34749797 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -98,20 +98,16 @@ func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) { // After the LDBStore.Put, it is ensured that the MemStore // contains the chunk with the same data, but nil ReqC channel. func (ls *LocalStore) Put(ctx context.Context, chunk *Chunk) { - if l := len(chunk.SData); l < 9 { - log.Debug("incomplete chunk data", "addr", chunk.Addr, "length", l) - chunk.SetErrored(ErrChunkInvalid) - chunk.markAsStored() - return - } valid := true + // ls.Validators contains a list of one validator per chunk type. + // if one validator succeeds, then the chunk is valid for _, v := range ls.Validators { if valid = v.Validate(chunk.Addr, chunk.SData); valid { break } } if !valid { - log.Trace("invalid content address", "addr", chunk.Addr) + log.Trace("invalid chunk", "addr", chunk.Addr, "len", len(chunk.SData)) chunk.SetErrored(ErrChunkInvalid) chunk.markAsStored() return diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 2bb81efa3a..ae62218fe8 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -20,6 +20,8 @@ import ( "io/ioutil" "os" "testing" + + "github.com/ethereum/go-ethereum/swarm/chunk" ) var ( @@ -61,7 +63,7 @@ func TestValidator(t *testing.T) { // add content address validator and check puts // bad should fail, good should pass store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc)) - chunks = GenerateRandomChunks(DefaultChunkSize, 2) + chunks = GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] copy(badChunk.SData, goodChunk.SData) @@ -79,7 +81,7 @@ func TestValidator(t *testing.T) { var negV boolTestValidator store.Validators = append(store.Validators, negV) - chunks = GenerateRandomChunks(DefaultChunkSize, 2) + chunks = GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] copy(badChunk.SData, goodChunk.SData) @@ -97,7 +99,7 @@ func TestValidator(t *testing.T) { var posV boolTestValidator = true store.Validators = append(store.Validators, posV) - chunks = GenerateRandomChunks(DefaultChunkSize, 2) + chunks = GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] copy(badChunk.SData, goodChunk.SData) diff --git a/swarm/storage/mru/handler.go b/swarm/storage/mru/handler.go index 32f43d5029..57561fd14b 100644 --- a/swarm/storage/mru/handler.go +++ b/swarm/storage/mru/handler.go @@ -21,17 +21,15 @@ package mru import ( "bytes" "context" - "fmt" "sync" "time" "unsafe" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage" ) -const chunkSize = 4096 // temporary until we implement FileStore in the resourcehandler - type Handler struct { chunkStore *storage.NetStore HashSize int @@ -66,8 +64,7 @@ func init() { } // NewHandler creates a new Mutable Resource API -func NewHandler(params *HandlerParams) (*Handler, error) { - +func NewHandler(params *HandlerParams) *Handler { rh := &Handler{ resources: make(map[uint64]*resource), storeTimeout: defaultStoreTimeout, @@ -82,7 +79,7 @@ func NewHandler(params *HandlerParams) (*Handler, error) { hashPool.Put(hashfunc) } - return rh, nil + return rh } // SetStore sets the store backend for the Mutable Resource API @@ -94,9 +91,8 @@ func (h *Handler) SetStore(store *storage.NetStore) { // If it looks like a resource update, the chunk address is checked against the ownerAddr of the update's signature // It implements the storage.ChunkValidator interface func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { - dataLength := len(data) - if dataLength < minimumChunkLength { + if dataLength < minimumChunkLength || dataLength > chunk.DefaultSize+8 { return false } @@ -106,7 +102,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { rootAddr, _ := metadataHash(data) valid := bytes.Equal(chunkAddr, rootAddr) if !valid { - log.Debug(fmt.Sprintf("Invalid root metadata chunk with address: %s", chunkAddr.Hex())) + log.Debug("Invalid root metadata chunk with address", "addr", chunkAddr.Hex()) } return valid } @@ -118,7 +114,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { // First, deserialize the chunk var r SignedResourceUpdate if err := r.fromChunk(chunkAddr, data); err != nil { - log.Debug("Invalid resource chunk with address %s: %s ", chunkAddr.Hex(), err.Error()) + log.Debug("Invalid resource chunk", "addr", chunkAddr.Hex(), "err", err.Error()) return false } @@ -126,7 +122,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { // that was used to retrieve this chunk // if this validation fails, someone forged a chunk. if !bytes.Equal(chunkAddr, r.updateHeader.UpdateAddr()) { - log.Debug("period,version,rootAddr contained in update chunk do not match updateAddr %s", chunkAddr.Hex()) + log.Debug("period,version,rootAddr contained in update chunk do not match updateAddr", "addr", chunkAddr.Hex()) return false } @@ -134,7 +130,7 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { // If it fails, it means either the signature is not valid, data is corrupted // or someone is trying to update someone else's resource. if err := r.Verify(); err != nil { - log.Debug("Invalid signature: %v", err) + log.Debug("Invalid signature", "err", err) return false } @@ -172,11 +168,6 @@ func (h *Handler) GetVersion(rootAddr storage.Address) (uint32, error) { return rsrc.version, nil } -// \TODO should be hashsize * branches from the chosen chunker, implement with FileStore -func (h *Handler) chunkSize() int64 { - return chunkSize -} - // New creates a new metadata chunk out of the request passed in. func (h *Handler) New(ctx context.Context, request *Request) error { diff --git a/swarm/storage/mru/resource_test.go b/swarm/storage/mru/resource_test.go index 95c9eccdfc..76d7c58a1e 100644 --- a/swarm/storage/mru/resource_test.go +++ b/swarm/storage/mru/resource_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/multihash" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -776,14 +777,11 @@ func TestValidatorInStore(t *testing.T) { // set up resource handler and add is as a validator to the localstore rhParams := &HandlerParams{} - rh, err := NewHandler(rhParams) - if err != nil { - t.Fatal(err) - } + rh := NewHandler(rhParams) store.Validators = append(store.Validators, rh) // create content addressed chunks, one good, one faulty - chunks := storage.GenerateRandomChunks(storage.DefaultChunkSize, 2) + chunks := storage.GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk := chunks[0] badChunk := chunks[1] badChunk.SData = goodChunk.SData diff --git a/swarm/storage/mru/testutil.go b/swarm/storage/mru/testutil.go index 751f51af32..6efcba9aba 100644 --- a/swarm/storage/mru/testutil.go +++ b/swarm/storage/mru/testutil.go @@ -38,10 +38,7 @@ func (t *TestHandler) Close() { // NewTestHandler creates Handler object to be used for testing purposes. func NewTestHandler(datadir string, params *HandlerParams) (*TestHandler, error) { path := filepath.Join(datadir, testDbDirName) - rh, err := NewHandler(params) - if err != nil { - return nil, fmt.Errorf("resource handler create fail: %v", err) - } + rh := NewHandler(params) localstoreparams := storage.NewDefaultLocalStoreParams() localstoreparams.Init(path) localStore, err := storage.NewLocalStore(localstoreparams, nil) diff --git a/swarm/storage/mru/update.go b/swarm/storage/mru/update.go index 88c4ac4e59..d1bd37ddff 100644 --- a/swarm/storage/mru/update.go +++ b/swarm/storage/mru/update.go @@ -20,6 +20,7 @@ import ( "encoding/binary" "errors" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/multihash" ) @@ -42,7 +43,7 @@ const chunkPrefixLength = 2 + 2 // // Minimum size is Header + 1 (minimum data length, enforced) const minimumUpdateDataLength = updateHeaderLength + 1 -const maxUpdateDataLength = chunkSize - signatureLength - updateHeaderLength - chunkPrefixLength +const maxUpdateDataLength = chunk.DefaultSize - signatureLength - updateHeaderLength - chunkPrefixLength // binaryPut serializes the resource update information into the given slice func (r *resourceUpdate) binaryPut(serializedData []byte) error { diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index 2923c81c5a..36ff66d045 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" ) @@ -101,11 +102,11 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get New chunks to store are store using the putter which the caller provides. */ func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) { - return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, DefaultChunkSize)).Split(ctx) + return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx) } func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) { - return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, DefaultChunkSize)).Append(ctx) + return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx) } // Entry to create a tree node diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 3114ef5767..53e3af485a 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/swarm/bmt" + "github.com/ethereum/go-ethereum/swarm/chunk" ) const MaxPO = 16 @@ -114,7 +115,9 @@ func MakeHashFunc(hash string) SwarmHasher { case "BMT": return func() SwarmHash { hasher := sha3.NewKeccak256 - pool := bmt.NewTreePool(hasher, bmt.SegmentCount, bmt.PoolSize) + hasherSize := hasher().Size() + segmentCount := chunk.DefaultSize / hasherSize + pool := bmt.NewTreePool(hasher, segmentCount, bmt.PoolSize) return bmt.New(pool) } } @@ -230,8 +233,8 @@ func GenerateRandomChunk(dataSize int64) *Chunk { func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk) { var i int hasher := MakeHashFunc(DefaultHash)() - if dataSize > DefaultChunkSize { - dataSize = DefaultChunkSize + if dataSize > chunk.DefaultSize { + dataSize = chunk.DefaultSize } for i = 0; i < count; i++ { @@ -345,6 +348,10 @@ func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator { // Validate that the given key is a valid content address for the given data func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool { + if l := len(data); l < 9 || l > chunk.DefaultSize+8 { + return false + } + hasher := v.Hasher() hasher.ResetWithLength(data[:8]) hasher.Write(data[8:]) diff --git a/swarm/swarm.go b/swarm/swarm.go index c380a376f6..f731ff33d7 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -195,18 +195,13 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e var resourceHandler *mru.Handler rhparams := &mru.HandlerParams{} - resourceHandler, err = mru.NewHandler(rhparams) - if err != nil { - return nil, err - } + resourceHandler = mru.NewHandler(rhparams) resourceHandler.SetStore(netStore) - var validators []storage.ChunkValidator - validators = append(validators, storage.NewContentAddressValidator(storage.MakeHashFunc(storage.DefaultHash))) - if resourceHandler != nil { - validators = append(validators, resourceHandler) + self.lstore.Validators = []storage.ChunkValidator{ + storage.NewContentAddressValidator(storage.MakeHashFunc(storage.DefaultHash)), + resourceHandler, } - self.lstore.Validators = validators // setup local store log.Debug(fmt.Sprintf("Set up local storage")) From e0e0e53401e93733d921338b6d794162c40a7883 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 14 Aug 2018 23:30:42 +0800 Subject: [PATCH 05/96] crypto: change formula for create2 (#17393) --- core/vm/evm.go | 2 +- crypto/crypto.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index a2722537dd..a24f6f3865 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -427,7 +427,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I // Create2 creates a new contract using code as deployment code. // -// The different between Create2 with Create is Create2 uses sha3(msg.sender ++ salt ++ init_code)[12:] +// The different between Create2 with Create is Create2 uses sha3(0xff ++ msg.sender ++ salt ++ sha3(init_code))[12:] // instead of the usual sender-and-nonce-hash as the address where the contract is initialized at. func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment *big.Int, salt *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { contractAddr = crypto.CreateAddress2(caller.Address(), common.BigToHash(salt), code) diff --git a/crypto/crypto.go b/crypto/crypto.go index dec6e3c19e..3211957e0a 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -78,8 +78,8 @@ func CreateAddress(b common.Address, nonce uint64) common.Address { // CreateAddress2 creates an ethereum address given the address bytes, initial // contract code and a salt. -func CreateAddress2(b common.Address, salt common.Hash, code []byte) common.Address { - return common.BytesToAddress(Keccak256([]byte{0xff}, b.Bytes(), salt.Bytes(), code)[12:]) +func CreateAddress2(b common.Address, salt [32]byte, code []byte) common.Address { + return common.BytesToAddress(Keccak256([]byte{0xff}, b.Bytes(), salt[:], Keccak256(code))[12:]) } // ToECDSA creates a private key with the given D value. From a1783d169732dd34aa8c7d68f411ce741c1a5015 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 14 Aug 2018 23:34:33 +0800 Subject: [PATCH 06/96] miner: move agent logic to worker (#17351) * miner: move agent logic to worker * miner: polish * core: persist block before reorg --- core/blockchain.go | 7 +- miner/agent.go | 116 ----- miner/miner.go | 64 +-- miner/worker.go | 1023 +++++++++++++++++++++++------------------- miner/worker_test.go | 212 +++++++++ 5 files changed, 797 insertions(+), 625 deletions(-) delete mode 100644 miner/agent.go create mode 100644 miner/worker_test.go diff --git a/core/blockchain.go b/core/blockchain.go index 62dc261250..0461da7fd9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -899,9 +899,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil { return NonStatTy, err } - // Write other block data using a batch. - batch := bc.db.NewBatch() - rawdb.WriteBlock(batch, block) + rawdb.WriteBlock(bc.db, block) root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) if err != nil { @@ -955,6 +953,9 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } } } + + // Write other block data using a batch. + batch := bc.db.NewBatch() rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) // If the total difficulty is higher than our known, add it to the canonical chain diff --git a/miner/agent.go b/miner/agent.go deleted file mode 100644 index e922ea153c..0000000000 --- a/miner/agent.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package miner - -import ( - "sync" - "sync/atomic" - - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/log" -) - -type CpuAgent struct { - mu sync.Mutex - - taskCh chan *Package - returnCh chan<- *Package - stop chan struct{} - quitCurrentOp chan struct{} - - chain consensus.ChainReader - engine consensus.Engine - - started int32 // started indicates whether the agent is currently started -} - -func NewCpuAgent(chain consensus.ChainReader, engine consensus.Engine) *CpuAgent { - agent := &CpuAgent{ - chain: chain, - engine: engine, - stop: make(chan struct{}, 1), - taskCh: make(chan *Package, 1), - } - return agent -} - -func (self *CpuAgent) AssignTask(p *Package) { - if atomic.LoadInt32(&self.started) == 1 { - self.taskCh <- p - } -} -func (self *CpuAgent) DeliverTo(ch chan<- *Package) { self.returnCh = ch } - -func (self *CpuAgent) Start() { - if !atomic.CompareAndSwapInt32(&self.started, 0, 1) { - return // agent already started - } - go self.update() -} - -func (self *CpuAgent) Stop() { - if !atomic.CompareAndSwapInt32(&self.started, 1, 0) { - return // agent already stopped - } - self.stop <- struct{}{} -done: - // Empty work channel - for { - select { - case <-self.taskCh: - default: - break done - } - } -} - -func (self *CpuAgent) update() { -out: - for { - select { - case p := <-self.taskCh: - self.mu.Lock() - if self.quitCurrentOp != nil { - close(self.quitCurrentOp) - } - self.quitCurrentOp = make(chan struct{}) - go self.mine(p, self.quitCurrentOp) - self.mu.Unlock() - case <-self.stop: - self.mu.Lock() - if self.quitCurrentOp != nil { - close(self.quitCurrentOp) - self.quitCurrentOp = nil - } - self.mu.Unlock() - break out - } - } -} - -func (self *CpuAgent) mine(p *Package, stop <-chan struct{}) { - var err error - if p.Block, err = self.engine.Seal(self.chain, p.Block, stop); p.Block != nil { - log.Info("Successfully sealed new block", "number", p.Block.Number(), "hash", p.Block.Hash()) - self.returnCh <- p - } else { - if err != nil { - log.Warn("Block sealing failed", "err", err) - } - self.returnCh <- nil - } -} diff --git a/miner/miner.go b/miner/miner.go index 4c5717c8ad..e350e456e9 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -21,14 +21,12 @@ import ( "fmt" "sync/atomic" - "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -36,10 +34,8 @@ import ( // Backend wraps all methods required for mining. type Backend interface { - AccountManager() *accounts.Manager BlockChain() *core.BlockChain TxPool() *core.TxPool - ChainDb() ethdb.Database } // Miner creates blocks and searches for proof-of-work values. @@ -49,6 +45,7 @@ type Miner struct { coinbase common.Address eth Backend engine consensus.Engine + exitCh chan struct{} canStart int32 // can start indicates whether we can start the mining operation shouldStart int32 // should start indicates whether we should start after sync @@ -59,10 +56,10 @@ func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine con eth: eth, mux: mux, engine: engine, + exitCh: make(chan struct{}), worker: newWorker(config, engine, eth, mux), canStart: 1, } - miner.Register(NewCpuAgent(eth.BlockChain(), engine)) go miner.update() return miner @@ -74,28 +71,35 @@ func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine con // and halt your mining operation for as long as the DOS continues. func (self *Miner) update() { events := self.mux.Subscribe(downloader.StartEvent{}, downloader.DoneEvent{}, downloader.FailedEvent{}) -out: - for ev := range events.Chan() { - switch ev.Data.(type) { - case downloader.StartEvent: - atomic.StoreInt32(&self.canStart, 0) - if self.Mining() { - self.Stop() - atomic.StoreInt32(&self.shouldStart, 1) - log.Info("Mining aborted due to sync") - } - case downloader.DoneEvent, downloader.FailedEvent: - shouldStart := atomic.LoadInt32(&self.shouldStart) == 1 + defer events.Unsubscribe() - atomic.StoreInt32(&self.canStart, 1) - atomic.StoreInt32(&self.shouldStart, 0) - if shouldStart { - self.Start(self.coinbase) + for { + select { + case ev := <-events.Chan(): + if ev == nil { + return } - // unsubscribe. we're only interested in this event once - events.Unsubscribe() - // stop immediately and ignore all further pending events - break out + switch ev.Data.(type) { + case downloader.StartEvent: + atomic.StoreInt32(&self.canStart, 0) + if self.Mining() { + self.Stop() + atomic.StoreInt32(&self.shouldStart, 1) + log.Info("Mining aborted due to sync") + } + case downloader.DoneEvent, downloader.FailedEvent: + shouldStart := atomic.LoadInt32(&self.shouldStart) == 1 + + atomic.StoreInt32(&self.canStart, 1) + atomic.StoreInt32(&self.shouldStart, 0) + if shouldStart { + self.Start(self.coinbase) + } + // stop immediately and ignore all further pending events + return + } + case <-self.exitCh: + return } } } @@ -109,7 +113,6 @@ func (self *Miner) Start(coinbase common.Address) { return } self.worker.start() - self.worker.commitNewWork() } func (self *Miner) Stop() { @@ -117,12 +120,9 @@ func (self *Miner) Stop() { atomic.StoreInt32(&self.shouldStart, 0) } -func (self *Miner) Register(agent Agent) { - self.worker.register(agent) -} - -func (self *Miner) Unregister(agent Agent) { - self.worker.unregister(agent) +func (self *Miner) Close() { + self.worker.close() + close(self.exitCh) } func (self *Miner) Mining() bool { diff --git a/miner/worker.go b/miner/worker.go index ae695f0198..81a63c29a4 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -32,16 +32,14 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) const ( - resultQueueSize = 10 - miningLogAtDepth = 5 - + // resultQueueSize is the size of channel listening to sealing result. + resultQueueSize = 10 // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 @@ -49,17 +47,10 @@ const ( chainHeadChanSize = 10 // chainSideChanSize is the size of channel listening to ChainSideEvent. chainSideChanSize = 10 + miningLogAtDepth = 5 ) -// Agent can register themselves with the worker -type Agent interface { - AssignTask(*Package) - DeliverTo(chan<- *Package) - Start() - Stop() -} - -// Env is the workers current environment and holds all of the current state information. +// Env is the worker's current environment and holds all of the current state information. type Env struct { config *params.ChainConfig signer types.Signer @@ -74,461 +65,20 @@ type Env struct { header *types.Header txs []*types.Transaction receipts []*types.Receipt - - createdAt time.Time } -// Package contains all information for consensus engine sealing and result submitting. -type Package struct { - Receipts []*types.Receipt - State *state.StateDB - Block *types.Block -} +func (env *Env) commitTransaction(tx *types.Transaction, bc *core.BlockChain, coinbase common.Address, gp *core.GasPool) (error, []*types.Log) { + snap := env.state.Snapshot() -// worker is the main object which takes care of applying messages to the new state -type worker struct { - config *params.ChainConfig - engine consensus.Engine - - mu sync.Mutex - - // update loop - mux *event.TypeMux - txsCh chan core.NewTxsEvent - txsSub event.Subscription - chainHeadCh chan core.ChainHeadEvent - chainHeadSub event.Subscription - chainSideCh chan core.ChainSideEvent - chainSideSub event.Subscription - - agents map[Agent]struct{} - recv chan *Package - - eth Backend - chain *core.BlockChain - proc core.Validator - chainDb ethdb.Database - - coinbase common.Address - extra []byte - - currentMu sync.Mutex - current *Env - - snapshotMu sync.RWMutex - snapshotBlock *types.Block - snapshotState *state.StateDB - - uncleMu sync.Mutex - possibleUncles map[common.Hash]*types.Block - - unconfirmed *unconfirmedBlocks // set of locally mined blocks pending canonicalness confirmations - - // atomic status counters - running int32 // The indicator whether the consensus engine is running or not. -} - -func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux) *worker { - worker := &worker{ - config: config, - engine: engine, - eth: eth, - mux: mux, - txsCh: make(chan core.NewTxsEvent, txChanSize), - chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), - chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), - chainDb: eth.ChainDb(), - recv: make(chan *Package, resultQueueSize), - chain: eth.BlockChain(), - proc: eth.BlockChain().Validator(), - possibleUncles: make(map[common.Hash]*types.Block), - agents: make(map[Agent]struct{}), - unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), - } - // Subscribe NewTxsEvent for tx pool - worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) - // Subscribe events for blockchain - worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) - worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) - go worker.update() - - go worker.wait() - worker.commitNewWork() - - return worker -} - -func (self *worker) setEtherbase(addr common.Address) { - self.mu.Lock() - defer self.mu.Unlock() - self.coinbase = addr -} - -func (self *worker) setExtra(extra []byte) { - self.mu.Lock() - defer self.mu.Unlock() - self.extra = extra -} - -func (self *worker) pending() (*types.Block, *state.StateDB) { - // return a snapshot to avoid contention on currentMu mutex - self.snapshotMu.RLock() - defer self.snapshotMu.RUnlock() - return self.snapshotBlock, self.snapshotState.Copy() -} - -func (self *worker) pendingBlock() *types.Block { - // return a snapshot to avoid contention on currentMu mutex - self.snapshotMu.RLock() - defer self.snapshotMu.RUnlock() - return self.snapshotBlock -} - -func (self *worker) start() { - self.mu.Lock() - defer self.mu.Unlock() - atomic.StoreInt32(&self.running, 1) - for agent := range self.agents { - agent.Start() - } -} - -func (self *worker) stop() { - self.mu.Lock() - defer self.mu.Unlock() - - atomic.StoreInt32(&self.running, 0) - for agent := range self.agents { - agent.Stop() - } -} - -func (self *worker) isRunning() bool { - return atomic.LoadInt32(&self.running) == 1 -} - -func (self *worker) register(agent Agent) { - self.mu.Lock() - defer self.mu.Unlock() - self.agents[agent] = struct{}{} - agent.DeliverTo(self.recv) - if self.isRunning() { - agent.Start() - } -} - -func (self *worker) unregister(agent Agent) { - self.mu.Lock() - defer self.mu.Unlock() - delete(self.agents, agent) - agent.Stop() -} - -func (self *worker) update() { - defer self.txsSub.Unsubscribe() - defer self.chainHeadSub.Unsubscribe() - defer self.chainSideSub.Unsubscribe() - - for { - // A real event arrived, process interesting content - select { - // Handle ChainHeadEvent - case <-self.chainHeadCh: - self.commitNewWork() - - // Handle ChainSideEvent - case ev := <-self.chainSideCh: - self.uncleMu.Lock() - self.possibleUncles[ev.Block.Hash()] = ev.Block - self.uncleMu.Unlock() - - // Handle NewTxsEvent - case ev := <-self.txsCh: - // Apply transactions to the pending state if we're not mining. - // - // Note all transactions received may not be continuous with transactions - // already included in the current mining block. These transactions will - // be automatically eliminated. - if !self.isRunning() { - self.currentMu.Lock() - txs := make(map[common.Address]types.Transactions) - for _, tx := range ev.Txs { - acc, _ := types.Sender(self.current.signer, tx) - txs[acc] = append(txs[acc], tx) - } - txset := types.NewTransactionsByPriceAndNonce(self.current.signer, txs) - self.current.commitTransactions(self.mux, txset, self.chain, self.coinbase) - self.updateSnapshot() - self.currentMu.Unlock() - } else { - // If we're mining, but nothing is being processed, wake on new transactions - if self.config.Clique != nil && self.config.Clique.Period == 0 { - self.commitNewWork() - } - } - - // System stopped - case <-self.txsSub.Err(): - return - case <-self.chainHeadSub.Err(): - return - case <-self.chainSideSub.Err(): - return - } - } -} - -func (self *worker) wait() { - for { - for result := range self.recv { - - if result == nil { - continue - } - block := result.Block - - // Update the block hash in all logs since it is now available and not when the - // receipt/log of individual transactions were created. - for _, r := range result.Receipts { - for _, l := range r.Logs { - l.BlockHash = block.Hash() - } - } - for _, log := range result.State.Logs() { - log.BlockHash = block.Hash() - } - self.currentMu.Lock() - stat, err := self.chain.WriteBlockWithState(block, result.Receipts, result.State) - self.currentMu.Unlock() - if err != nil { - log.Error("Failed writing block to chain", "err", err) - continue - } - // Broadcast the block and announce chain insertion event - self.mux.Post(core.NewMinedBlockEvent{Block: block}) - var ( - events []interface{} - logs = result.State.Logs() - ) - events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) - if stat == core.CanonStatTy { - events = append(events, core.ChainHeadEvent{Block: block}) - } - self.chain.PostChainEvents(events, logs) - - // Insert the block into the set of pending ones to wait for confirmations - self.unconfirmed.Insert(block.NumberU64(), block.Hash()) - } - } -} - -// push sends a new work task to currently live miner agents. -func (self *worker) push(p *Package) { - for agent := range self.agents { - agent.AssignTask(p) - } -} - -// makeCurrent creates a new environment for the current cycle. -func (self *worker) makeCurrent(parent *types.Block, header *types.Header) error { - state, err := self.chain.StateAt(parent.Root()) + receipt, _, err := core.ApplyTransaction(env.config, bc, &coinbase, gp, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) if err != nil { - return err - } - env := &Env{ - config: self.config, - signer: types.NewEIP155Signer(self.config.ChainID), - state: state, - ancestors: mapset.NewSet(), - family: mapset.NewSet(), - uncles: mapset.NewSet(), - header: header, - createdAt: time.Now(), + env.state.RevertToSnapshot(snap) + return err, nil } + env.txs = append(env.txs, tx) + env.receipts = append(env.receipts, receipt) - // when 08 is processed ancestors contain 07 (quick block) - for _, ancestor := range self.chain.GetBlocksFromHash(parent.Hash(), 7) { - for _, uncle := range ancestor.Uncles() { - env.family.Add(uncle.Hash()) - } - env.family.Add(ancestor.Hash()) - env.ancestors.Add(ancestor.Hash()) - } - - // Keep track of transactions which return errors so they can be removed - env.tcount = 0 - self.current = env - return nil -} - -func (self *worker) commitNewWork() { - self.mu.Lock() - defer self.mu.Unlock() - self.uncleMu.Lock() - defer self.uncleMu.Unlock() - self.currentMu.Lock() - defer self.currentMu.Unlock() - - tstart := time.Now() - parent := self.chain.CurrentBlock() - - tstamp := tstart.Unix() - if parent.Time().Cmp(new(big.Int).SetInt64(tstamp)) >= 0 { - tstamp = parent.Time().Int64() + 1 - } - // this will ensure we're not going off too far in the future - if now := time.Now().Unix(); tstamp > now+1 { - wait := time.Duration(tstamp-now) * time.Second - log.Info("Mining too far in the future", "wait", common.PrettyDuration(wait)) - time.Sleep(wait) - } - - num := parent.Number() - header := &types.Header{ - ParentHash: parent.Hash(), - Number: num.Add(num, common.Big1), - GasLimit: core.CalcGasLimit(parent), - Extra: self.extra, - Time: big.NewInt(tstamp), - } - // Only set the coinbase if our consensus engine is running (avoid spurious block rewards) - if self.isRunning() { - if self.coinbase == (common.Address{}) { - log.Error("Refusing to mine without etherbase") - return - } - header.Coinbase = self.coinbase - } - if err := self.engine.Prepare(self.chain, header); err != nil { - log.Error("Failed to prepare header for mining", "err", err) - return - } - // If we are care about TheDAO hard-fork check whether to override the extra-data or not - if daoBlock := self.config.DAOForkBlock; daoBlock != nil { - // Check whether the block is among the fork extra-override range - limit := new(big.Int).Add(daoBlock, params.DAOForkExtraRange) - if header.Number.Cmp(daoBlock) >= 0 && header.Number.Cmp(limit) < 0 { - // Depending whether we support or oppose the fork, override differently - if self.config.DAOForkSupport { - header.Extra = common.CopyBytes(params.DAOForkBlockExtra) - } else if bytes.Equal(header.Extra, params.DAOForkBlockExtra) { - header.Extra = []byte{} // If miner opposes, don't let it use the reserved extra-data - } - } - } - // Could potentially happen if starting to mine in an odd state. - err := self.makeCurrent(parent, header) - if err != nil { - log.Error("Failed to create mining context", "err", err) - return - } - // Create the current work task and check any fork transitions needed - env := self.current - if self.config.DAOForkSupport && self.config.DAOForkBlock != nil && self.config.DAOForkBlock.Cmp(header.Number) == 0 { - misc.ApplyDAOHardFork(env.state) - } - - // compute uncles for the new block. - var ( - uncles []*types.Header - badUncles []common.Hash - ) - for hash, uncle := range self.possibleUncles { - if len(uncles) == 2 { - break - } - if err := self.commitUncle(env, uncle.Header()); err != nil { - log.Trace("Bad uncle found and will be removed", "hash", hash) - log.Trace(fmt.Sprint(uncle)) - - badUncles = append(badUncles, hash) - } else { - log.Debug("Committing new uncle to block", "hash", hash) - uncles = append(uncles, uncle.Header()) - } - } - for _, hash := range badUncles { - delete(self.possibleUncles, hash) - } - - var ( - emptyBlock *types.Block - fullBlock *types.Block - ) - - // Create an empty block based on temporary copied state for sealing in advance without waiting block - // execution finished. - emptyState := env.state.Copy() - if emptyBlock, err = self.engine.Finalize(self.chain, header, emptyState, nil, uncles, nil); err != nil { - log.Error("Failed to finalize block for temporary sealing", "err", err) - } else { - // Push empty work in advance without applying pending transaction. - // The reason is transactions execution can cost a lot and sealer need to - // take advantage of this part time. - if self.isRunning() { - log.Info("Commit new empty mining work", "number", emptyBlock.Number(), "uncles", len(uncles)) - self.push(&Package{nil, emptyState, emptyBlock}) - } - } - - // Fill the block with all available pending transactions. - pending, err := self.eth.TxPool().Pending() - if err != nil { - log.Error("Failed to fetch pending transactions", "err", err) - return - } - txs := types.NewTransactionsByPriceAndNonce(self.current.signer, pending) - env.commitTransactions(self.mux, txs, self.chain, self.coinbase) - - // Create the full block to seal with the consensus engine - if fullBlock, err = self.engine.Finalize(self.chain, header, env.state, env.txs, uncles, env.receipts); err != nil { - log.Error("Failed to finalize block for sealing", "err", err) - return - } - // We only care about logging if we're actually mining. - if self.isRunning() { - log.Info("Commit new full mining work", "number", fullBlock.Number(), "txs", env.tcount, "uncles", len(uncles), "elapsed", common.PrettyDuration(time.Since(tstart))) - self.unconfirmed.Shift(fullBlock.NumberU64() - 1) - self.push(&Package{env.receipts, env.state, fullBlock}) - } - self.updateSnapshot() -} - -func (self *worker) commitUncle(env *Env, uncle *types.Header) error { - hash := uncle.Hash() - if env.uncles.Contains(hash) { - return fmt.Errorf("uncle not unique") - } - if !env.ancestors.Contains(uncle.ParentHash) { - return fmt.Errorf("uncle's parent unknown (%x)", uncle.ParentHash[0:4]) - } - if env.family.Contains(hash) { - return fmt.Errorf("uncle already in family (%x)", hash) - } - env.uncles.Add(uncle.Hash()) - return nil -} - -func (self *worker) updateSnapshot() { - self.snapshotMu.Lock() - defer self.snapshotMu.Unlock() - - var uncles []*types.Header - self.current.uncles.Each(func(item interface{}) bool { - if header, ok := item.(*types.Header); ok { - uncles = append(uncles, header) - return true - } - return false - }) - - self.snapshotBlock = types.NewBlock( - self.current.header, - self.current.txs, - uncles, - self.current.receipts, - ) - self.snapshotState = self.current.state.Copy() + return nil, receipt.Logs } func (env *Env) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) { @@ -616,16 +166,541 @@ func (env *Env) commitTransactions(mux *event.TypeMux, txs *types.TransactionsBy } } -func (env *Env) commitTransaction(tx *types.Transaction, bc *core.BlockChain, coinbase common.Address, gp *core.GasPool) (error, []*types.Log) { - snap := env.state.Snapshot() - - receipt, _, err := core.ApplyTransaction(env.config, bc, &coinbase, gp, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) - if err != nil { - env.state.RevertToSnapshot(snap) - return err, nil - } - env.txs = append(env.txs, tx) - env.receipts = append(env.receipts, receipt) - - return nil, receipt.Logs +// task contains all information for consensus engine sealing and result submitting. +type task struct { + receipts []*types.Receipt + state *state.StateDB + block *types.Block + createdAt time.Time +} + +// worker is the main object which takes care of submitting new work to consensus engine +// and gathering the sealing result. +type worker struct { + config *params.ChainConfig + engine consensus.Engine + eth Backend + chain *core.BlockChain + + // Subscriptions + mux *event.TypeMux + txsCh chan core.NewTxsEvent + txsSub event.Subscription + chainHeadCh chan core.ChainHeadEvent + chainHeadSub event.Subscription + chainSideCh chan core.ChainSideEvent + chainSideSub event.Subscription + + // Channels + newWork chan struct{} + taskCh chan *task + resultCh chan *task + exitCh chan struct{} + + current *Env // An environment for current running cycle. + possibleUncles map[common.Hash]*types.Block // A set of side blocks as the possible uncle blocks. + unconfirmed *unconfirmedBlocks // A set of locally mined blocks pending canonicalness confirmations. + + mu sync.RWMutex // The lock used to protect the coinbase and extra fields + coinbase common.Address + extra []byte + + snapshotMu sync.RWMutex // The lock used to protect the block snapshot and state snapshot + snapshotBlock *types.Block + snapshotState *state.StateDB + + // atomic status counters + running int32 // The indicator whether the consensus engine is running or not. + + // Test hooks + newTaskHook func(*task) // Method to call upon receiving a new sealing task + fullTaskInterval func() // Method to call before pushing the full sealing task +} + +func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux) *worker { + worker := &worker{ + config: config, + engine: engine, + eth: eth, + mux: mux, + chain: eth.BlockChain(), + possibleUncles: make(map[common.Hash]*types.Block), + unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), + txsCh: make(chan core.NewTxsEvent, txChanSize), + chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), + chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), + newWork: make(chan struct{}, 1), + taskCh: make(chan *task), + resultCh: make(chan *task, resultQueueSize), + exitCh: make(chan struct{}), + } + // Subscribe NewTxsEvent for tx pool + worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) + // Subscribe events for blockchain + worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) + worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) + + go worker.mainLoop() + go worker.resultLoop() + go worker.taskLoop() + + // Submit first work to initialize pending state. + worker.newWork <- struct{}{} + return worker +} + +// setEtherbase sets the etherbase used to initialize the block coinbase field. +func (w *worker) setEtherbase(addr common.Address) { + w.mu.Lock() + defer w.mu.Unlock() + w.coinbase = addr +} + +// setExtra sets the content used to initialize the block extra field. +func (w *worker) setExtra(extra []byte) { + w.mu.Lock() + defer w.mu.Unlock() + w.extra = extra +} + +// pending returns the pending state and corresponding block. +func (w *worker) pending() (*types.Block, *state.StateDB) { + // return a snapshot to avoid contention on currentMu mutex + w.snapshotMu.RLock() + defer w.snapshotMu.RUnlock() + if w.snapshotState == nil { + return nil, nil + } + return w.snapshotBlock, w.snapshotState.Copy() +} + +// pendingBlock returns pending block. +func (w *worker) pendingBlock() *types.Block { + // return a snapshot to avoid contention on currentMu mutex + w.snapshotMu.RLock() + defer w.snapshotMu.RUnlock() + return w.snapshotBlock +} + +// start sets the running status as 1 and triggers new work submitting. +func (w *worker) start() { + atomic.StoreInt32(&w.running, 1) + w.newWork <- struct{}{} +} + +// stop sets the running status as 0. +func (w *worker) stop() { + atomic.StoreInt32(&w.running, 0) +} + +// isRunning returns an indicator whether worker is running or not. +func (w *worker) isRunning() bool { + return atomic.LoadInt32(&w.running) == 1 +} + +// close terminates all background threads maintained by the worker and cleans up buffered channels. +// Note the worker does not support being closed multiple times. +func (w *worker) close() { + close(w.exitCh) + // Clean up buffered channels + for empty := false; !empty; { + select { + case <-w.resultCh: + default: + empty = true + } + } +} + +// mainLoop is a standalone goroutine to regenerate the sealing task based on the received event. +func (w *worker) mainLoop() { + defer w.txsSub.Unsubscribe() + defer w.chainHeadSub.Unsubscribe() + defer w.chainSideSub.Unsubscribe() + + for { + select { + case <-w.newWork: + // Submit a work when the worker is created or started. + w.commitNewWork() + + case <-w.chainHeadCh: + // Resubmit a work for new cycle once worker receives chain head event. + w.commitNewWork() + + case ev := <-w.chainSideCh: + // Add side block to possible uncle block set. + w.possibleUncles[ev.Block.Hash()] = ev.Block + + case ev := <-w.txsCh: + // Apply transactions to the pending state if we're not mining. + // + // Note all transactions received may not be continuous with transactions + // already included in the current mining block. These transactions will + // be automatically eliminated. + if !w.isRunning() && w.current != nil { + w.mu.Lock() + coinbase := w.coinbase + w.mu.Unlock() + + txs := make(map[common.Address]types.Transactions) + for _, tx := range ev.Txs { + acc, _ := types.Sender(w.current.signer, tx) + txs[acc] = append(txs[acc], tx) + } + txset := types.NewTransactionsByPriceAndNonce(w.current.signer, txs) + w.current.commitTransactions(w.mux, txset, w.chain, coinbase) + w.updateSnapshot() + } else { + // If we're mining, but nothing is being processed, wake on new transactions + if w.config.Clique != nil && w.config.Clique.Period == 0 { + w.commitNewWork() + } + } + + // System stopped + case <-w.exitCh: + return + case <-w.txsSub.Err(): + return + case <-w.chainHeadSub.Err(): + return + case <-w.chainSideSub.Err(): + return + } + } +} + +// seal pushes a sealing task to consensus engine and submits the result. +func (w *worker) seal(t *task, stop <-chan struct{}) { + var ( + err error + res *task + ) + + if t.block, err = w.engine.Seal(w.chain, t.block, stop); t.block != nil { + log.Info("Successfully sealed new block", "number", t.block.Number(), "hash", t.block.Hash(), + "elapsed", common.PrettyDuration(time.Since(t.createdAt))) + res = t + } else { + if err != nil { + log.Warn("Block sealing failed", "err", err) + } + res = nil + } + select { + case w.resultCh <- res: + case <-w.exitCh: + } +} + +// taskLoop is a standalone goroutine to fetch sealing task from the generator and +// push them to consensus engine. +func (w *worker) taskLoop() { + var stopCh chan struct{} + + // interrupt aborts the in-flight sealing task. + interrupt := func() { + if stopCh != nil { + close(stopCh) + stopCh = nil + } + } + for { + select { + case task := <-w.taskCh: + if w.newTaskHook != nil { + w.newTaskHook(task) + } + interrupt() + stopCh = make(chan struct{}) + go w.seal(task, stopCh) + case <-w.exitCh: + interrupt() + return + } + } +} + +// resultLoop is a standalone goroutine to handle sealing result submitting +// and flush relative data to the database. +func (w *worker) resultLoop() { + for { + select { + case result := <-w.resultCh: + if result == nil { + continue + } + block := result.block + + // Update the block hash in all logs since it is now available and not when the + // receipt/log of individual transactions were created. + for _, r := range result.receipts { + for _, l := range r.Logs { + l.BlockHash = block.Hash() + } + } + for _, log := range result.state.Logs() { + log.BlockHash = block.Hash() + } + // Commit block and state to database. + stat, err := w.chain.WriteBlockWithState(block, result.receipts, result.state) + if err != nil { + log.Error("Failed writing block to chain", "err", err) + continue + } + // Broadcast the block and announce chain insertion event + w.mux.Post(core.NewMinedBlockEvent{Block: block}) + var ( + events []interface{} + logs = result.state.Logs() + ) + switch stat { + case core.CanonStatTy: + events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) + events = append(events, core.ChainHeadEvent{Block: block}) + case core.SideStatTy: + events = append(events, core.ChainSideEvent{Block: block}) + } + w.chain.PostChainEvents(events, logs) + + // Insert the block into the set of pending ones to resultLoop for confirmations + w.unconfirmed.Insert(block.NumberU64(), block.Hash()) + + case <-w.exitCh: + return + } + } +} + +// makeCurrent creates a new environment for the current cycle. +func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { + state, err := w.chain.StateAt(parent.Root()) + if err != nil { + return err + } + env := &Env{ + config: w.config, + signer: types.NewEIP155Signer(w.config.ChainID), + state: state, + ancestors: mapset.NewSet(), + family: mapset.NewSet(), + uncles: mapset.NewSet(), + header: header, + } + + // when 08 is processed ancestors contain 07 (quick block) + for _, ancestor := range w.chain.GetBlocksFromHash(parent.Hash(), 7) { + for _, uncle := range ancestor.Uncles() { + env.family.Add(uncle.Hash()) + } + env.family.Add(ancestor.Hash()) + env.ancestors.Add(ancestor.Hash()) + } + + // Keep track of transactions which return errors so they can be removed + env.tcount = 0 + w.current = env + return nil +} + +// commitUncle adds the given block to uncle block set, returns error if failed to add. +func (w *worker) commitUncle(env *Env, uncle *types.Header) error { + hash := uncle.Hash() + if env.uncles.Contains(hash) { + return fmt.Errorf("uncle not unique") + } + if !env.ancestors.Contains(uncle.ParentHash) { + return fmt.Errorf("uncle's parent unknown (%x)", uncle.ParentHash[0:4]) + } + if env.family.Contains(hash) { + return fmt.Errorf("uncle already in family (%x)", hash) + } + env.uncles.Add(uncle.Hash()) + return nil +} + +// updateSnapshot updates pending snapshot block and state. +// Note this function assumes the current variable is thread safe. +func (w *worker) updateSnapshot() { + w.snapshotMu.Lock() + defer w.snapshotMu.Unlock() + + var uncles []*types.Header + w.current.uncles.Each(func(item interface{}) bool { + hash, ok := item.(common.Hash) + if !ok { + return false + } + uncle, exist := w.possibleUncles[hash] + if !exist { + return false + } + uncles = append(uncles, uncle.Header()) + return true + }) + + w.snapshotBlock = types.NewBlock( + w.current.header, + w.current.txs, + uncles, + w.current.receipts, + ) + + w.snapshotState = w.current.state.Copy() +} + +// commitNewWork generates several new sealing tasks based on the parent block. +func (w *worker) commitNewWork() { + w.mu.RLock() + defer w.mu.RUnlock() + + tstart := time.Now() + parent := w.chain.CurrentBlock() + + tstamp := tstart.Unix() + if parent.Time().Cmp(new(big.Int).SetInt64(tstamp)) >= 0 { + tstamp = parent.Time().Int64() + 1 + } + // this will ensure we're not going off too far in the future + if now := time.Now().Unix(); tstamp > now+1 { + wait := time.Duration(tstamp-now) * time.Second + log.Info("Mining too far in the future", "wait", common.PrettyDuration(wait)) + time.Sleep(wait) + } + + num := parent.Number() + header := &types.Header{ + ParentHash: parent.Hash(), + Number: num.Add(num, common.Big1), + GasLimit: core.CalcGasLimit(parent), + Extra: w.extra, + Time: big.NewInt(tstamp), + } + // Only set the coinbase if our consensus engine is running (avoid spurious block rewards) + if w.isRunning() { + if w.coinbase == (common.Address{}) { + log.Error("Refusing to mine without etherbase") + return + } + header.Coinbase = w.coinbase + } + if err := w.engine.Prepare(w.chain, header); err != nil { + log.Error("Failed to prepare header for mining", "err", err) + return + } + // If we are care about TheDAO hard-fork check whether to override the extra-data or not + if daoBlock := w.config.DAOForkBlock; daoBlock != nil { + // Check whether the block is among the fork extra-override range + limit := new(big.Int).Add(daoBlock, params.DAOForkExtraRange) + if header.Number.Cmp(daoBlock) >= 0 && header.Number.Cmp(limit) < 0 { + // Depending whether we support or oppose the fork, override differently + if w.config.DAOForkSupport { + header.Extra = common.CopyBytes(params.DAOForkBlockExtra) + } else if bytes.Equal(header.Extra, params.DAOForkBlockExtra) { + header.Extra = []byte{} // If miner opposes, don't let it use the reserved extra-data + } + } + } + // Could potentially happen if starting to mine in an odd state. + err := w.makeCurrent(parent, header) + if err != nil { + log.Error("Failed to create mining context", "err", err) + return + } + // Create the current work task and check any fork transitions needed + env := w.current + if w.config.DAOForkSupport && w.config.DAOForkBlock != nil && w.config.DAOForkBlock.Cmp(header.Number) == 0 { + misc.ApplyDAOHardFork(env.state) + } + + // compute uncles for the new block. + var ( + uncles []*types.Header + badUncles []common.Hash + ) + for hash, uncle := range w.possibleUncles { + if len(uncles) == 2 { + break + } + if err := w.commitUncle(env, uncle.Header()); err != nil { + log.Trace("Bad uncle found and will be removed", "hash", hash) + log.Trace(fmt.Sprint(uncle)) + + badUncles = append(badUncles, hash) + } else { + log.Debug("Committing new uncle to block", "hash", hash) + uncles = append(uncles, uncle.Header()) + } + } + for _, hash := range badUncles { + delete(w.possibleUncles, hash) + } + + var ( + emptyBlock, fullBlock *types.Block + emptyState, fullState *state.StateDB + ) + + // Create an empty block based on temporary copied state for sealing in advance without waiting block + // execution finished. + emptyState = env.state.Copy() + if emptyBlock, err = w.engine.Finalize(w.chain, header, emptyState, nil, uncles, nil); err != nil { + log.Error("Failed to finalize block for temporary sealing", "err", err) + } else { + // Push empty work in advance without applying pending transaction. + // The reason is transactions execution can cost a lot and sealer need to + // take advantage of this part time. + if w.isRunning() { + select { + case w.taskCh <- &task{receipts: nil, state: emptyState, block: emptyBlock, createdAt: time.Now()}: + log.Info("Commit new empty mining work", "number", emptyBlock.Number(), "uncles", len(uncles)) + case <-w.exitCh: + log.Info("Worker has exited") + return + } + } + } + + // Fill the block with all available pending transactions. + pending, err := w.eth.TxPool().Pending() + if err != nil { + log.Error("Failed to fetch pending transactions", "err", err) + return + } + // Short circuit if there is no available pending transactions + if len(pending) == 0 { + w.updateSnapshot() + return + } + txs := types.NewTransactionsByPriceAndNonce(w.current.signer, pending) + env.commitTransactions(w.mux, txs, w.chain, w.coinbase) + + // Create the full block to seal with the consensus engine + fullState = env.state.Copy() + if fullBlock, err = w.engine.Finalize(w.chain, header, fullState, env.txs, uncles, env.receipts); err != nil { + log.Error("Failed to finalize block for sealing", "err", err) + return + } + // Deep copy receipts here to avoid interaction between different tasks. + cpy := make([]*types.Receipt, len(env.receipts)) + for i, l := range env.receipts { + cpy[i] = new(types.Receipt) + *cpy[i] = *l + } + // We only care about logging if we're actually mining. + if w.isRunning() { + if w.fullTaskInterval != nil { + w.fullTaskInterval() + } + + select { + case w.taskCh <- &task{receipts: cpy, state: fullState, block: fullBlock, createdAt: time.Now()}: + w.unconfirmed.Shift(fullBlock.NumberU64() - 1) + log.Info("Commit new full mining work", "number", fullBlock.Number(), "txs", env.tcount, "uncles", len(uncles), "elapsed", common.PrettyDuration(time.Since(tstart))) + case <-w.exitCh: + log.Info("Worker has exited") + } + } + w.updateSnapshot() } diff --git a/miner/worker_test.go b/miner/worker_test.go new file mode 100644 index 0000000000..5823a608ef --- /dev/null +++ b/miner/worker_test.go @@ -0,0 +1,212 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package miner + +import ( + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/clique" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" +) + +var ( + // Test chain configurations + testTxPoolConfig core.TxPoolConfig + ethashChainConfig *params.ChainConfig + cliqueChainConfig *params.ChainConfig + + // Test accounts + testBankKey, _ = crypto.GenerateKey() + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) + testBankFunds = big.NewInt(1000000000000000000) + + acc1Key, _ = crypto.GenerateKey() + acc1Addr = crypto.PubkeyToAddress(acc1Key.PublicKey) + + // Test transactions + pendingTxs []*types.Transaction + newTxs []*types.Transaction +) + +func init() { + testTxPoolConfig = core.DefaultTxPoolConfig + testTxPoolConfig.Journal = "" + ethashChainConfig = params.TestChainConfig + cliqueChainConfig = params.TestChainConfig + cliqueChainConfig.Clique = ¶ms.CliqueConfig{ + Period: 1, + Epoch: 30000, + } + tx1, _ := types.SignTx(types.NewTransaction(0, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + pendingTxs = append(pendingTxs, tx1) + tx2, _ := types.SignTx(types.NewTransaction(1, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + newTxs = append(newTxs, tx2) +} + +// testWorkerBackend implements worker.Backend interfaces and wraps all information needed during the testing. +type testWorkerBackend struct { + db ethdb.Database + txPool *core.TxPool + chain *core.BlockChain + testTxFeed event.Feed +} + +func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) *testWorkerBackend { + var ( + db = ethdb.NewMemDatabase() + gspec = core.Genesis{ + Config: chainConfig, + Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + } + ) + + switch engine.(type) { + case *clique.Clique: + gspec.ExtraData = make([]byte, 32+common.AddressLength+65) + copy(gspec.ExtraData[32:], testBankAddress[:]) + case *ethash.Ethash: + default: + t.Fatal("unexpect consensus engine type") + } + gspec.MustCommit(db) + + chain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) + txpool := core.NewTxPool(testTxPoolConfig, chainConfig, chain) + + return &testWorkerBackend{ + db: db, + chain: chain, + txPool: txpool, + } +} + +func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } +func (b *testWorkerBackend) TxPool() *core.TxPool { return b.txPool } +func (b *testWorkerBackend) PostChainEvents(events []interface{}) { + b.chain.PostChainEvents(events, nil) +} + +func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) (*worker, *testWorkerBackend) { + backend := newTestWorkerBackend(t, chainConfig, engine) + backend.txPool.AddLocals(pendingTxs) + w := newWorker(chainConfig, engine, backend, new(event.TypeMux)) + w.setEtherbase(testBankAddress) + return w, backend +} + +func TestPendingStateAndBlockEthash(t *testing.T) { + testPendingStateAndBlock(t, ethashChainConfig, ethash.NewFaker()) +} +func TestPendingStateAndBlockClique(t *testing.T) { + testPendingStateAndBlock(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) +} + +func testPendingStateAndBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { + defer engine.Close() + + w, b := newTestWorker(t, chainConfig, engine) + defer w.close() + + // Ensure snapshot has been updated. + time.Sleep(100 * time.Millisecond) + block, state := w.pending() + if block.NumberU64() != 1 { + t.Errorf("block number mismatch, has %d, want %d", block.NumberU64(), 1) + } + if balance := state.GetBalance(acc1Addr); balance.Cmp(big.NewInt(1000)) != 0 { + t.Errorf("account balance mismatch, has %d, want %d", balance, 1000) + } + b.txPool.AddLocals(newTxs) + // Ensure the new tx events has been processed + time.Sleep(100 * time.Millisecond) + block, state = w.pending() + if balance := state.GetBalance(acc1Addr); balance.Cmp(big.NewInt(2000)) != 0 { + t.Errorf("account balance mismatch, has %d, want %d", balance, 2000) + } +} + +func TestEmptyWorkEthash(t *testing.T) { + testEmptyWork(t, ethashChainConfig, ethash.NewFaker()) +} +func TestEmptyWorkClique(t *testing.T) { + testEmptyWork(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) +} + +func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { + defer engine.Close() + + w, _ := newTestWorker(t, chainConfig, engine) + defer w.close() + + var ( + taskCh = make(chan struct{}, 2) + taskIndex int + ) + + checkEqual := func(t *testing.T, task *task, index int) { + receiptLen, balance := 0, big.NewInt(0) + if index == 1 { + receiptLen, balance = 1, big.NewInt(1000) + } + if len(task.receipts) != receiptLen { + t.Errorf("receipt number mismatch has %d, want %d", len(task.receipts), receiptLen) + } + if task.state.GetBalance(acc1Addr).Cmp(balance) != 0 { + t.Errorf("account balance mismatch has %d, want %d", task.state.GetBalance(acc1Addr), balance) + } + } + + w.newTaskHook = func(task *task) { + if task.block.NumberU64() == 1 { + checkEqual(t, task, taskIndex) + taskIndex += 1 + taskCh <- struct{}{} + } + } + w.fullTaskInterval = func() { + time.Sleep(100 * time.Millisecond) + } + + // Ensure worker has finished initialization + for { + b := w.pendingBlock() + if b != nil && b.NumberU64() == 1 { + break + } + } + + w.start() + for i := 0; i < 2; i += 1 { + to := time.NewTimer(time.Second) + select { + case <-taskCh: + case <-to.C: + t.Error("new task timeout") + } + } +} From b2ddb1fcbf77771d0693ee5a00f8ae1cd4c0f87c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 14 Aug 2018 22:44:46 +0200 Subject: [PATCH 07/96] les: implement client connection logic (#16899) This PR implements les.freeClientPool. It also adds a simulated clock in common/mclock, which enables time-sensitive tests to run quickly and still produce accurate results, and package common/prque which is a generalised variant of prque that enables removing elements other than the top one from the queue. les.freeClientPool implements a client database that limits the connection time of each client and manages accepting/rejecting incoming connections and even kicking out some connected clients. The pool calculates recent usage time for each known client (a value that increases linearly when the client is connected and decreases exponentially when not connected). Clients with lower recent usage are preferred, unknown nodes have the highest priority. Already connected nodes receive a small bias in their favor in order to avoid accepting and instantly kicking out clients. Note: the pool can use any string for client identification. Using signature keys for that purpose would not make sense when being known has a negative value for the client. Currently the LES protocol manager uses IP addresses (without port address) to identify clients. --- common/mclock/mclock.go | 31 +++++ common/mclock/simclock.go | 129 ++++++++++++++++++ common/prque/prque.go | 57 ++++++++ common/prque/sstack.go | 106 +++++++++++++++ les/freeclient.go | 278 ++++++++++++++++++++++++++++++++++++++ les/freeclient_test.go | 139 +++++++++++++++++++ les/handler.go | 22 ++- 7 files changed, 761 insertions(+), 1 deletion(-) create mode 100644 common/mclock/simclock.go create mode 100755 common/prque/prque.go create mode 100755 common/prque/sstack.go create mode 100644 les/freeclient.go create mode 100644 les/freeclient_test.go diff --git a/common/mclock/mclock.go b/common/mclock/mclock.go index 02608d17b0..dcac59c6ce 100644 --- a/common/mclock/mclock.go +++ b/common/mclock/mclock.go @@ -30,3 +30,34 @@ type AbsTime time.Duration func Now() AbsTime { return AbsTime(monotime.Now()) } + +// Add returns t + d. +func (t AbsTime) Add(d time.Duration) AbsTime { + return t + AbsTime(d) +} + +// Clock interface makes it possible to replace the monotonic system clock with +// a simulated clock. +type Clock interface { + Now() AbsTime + Sleep(time.Duration) + After(time.Duration) <-chan time.Time +} + +// System implements Clock using the system clock. +type System struct{} + +// Now implements Clock. +func (System) Now() AbsTime { + return AbsTime(monotime.Now()) +} + +// Sleep implements Clock. +func (System) Sleep(d time.Duration) { + time.Sleep(d) +} + +// After implements Clock. +func (System) After(d time.Duration) <-chan time.Time { + return time.After(d) +} diff --git a/common/mclock/simclock.go b/common/mclock/simclock.go new file mode 100644 index 0000000000..e014f56150 --- /dev/null +++ b/common/mclock/simclock.go @@ -0,0 +1,129 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package mclock + +import ( + "sync" + "time" +) + +// Simulated implements a virtual Clock for reproducible time-sensitive tests. It +// simulates a scheduler on a virtual timescale where actual processing takes zero time. +// +// The virtual clock doesn't advance on its own, call Run to advance it and execute timers. +// Since there is no way to influence the Go scheduler, testing timeout behaviour involving +// goroutines needs special care. A good way to test such timeouts is as follows: First +// perform the action that is supposed to time out. Ensure that the timer you want to test +// is created. Then run the clock until after the timeout. Finally observe the effect of +// the timeout using a channel or semaphore. +type Simulated struct { + now AbsTime + scheduled []event + mu sync.RWMutex + cond *sync.Cond +} + +type event struct { + do func() + at AbsTime +} + +// Run moves the clock by the given duration, executing all timers before that duration. +func (s *Simulated) Run(d time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + s.init() + + end := s.now + AbsTime(d) + for len(s.scheduled) > 0 { + ev := s.scheduled[0] + if ev.at > end { + break + } + s.now = ev.at + ev.do() + s.scheduled = s.scheduled[1:] + } + s.now = end +} + +func (s *Simulated) ActiveTimers() int { + s.mu.RLock() + defer s.mu.RUnlock() + + return len(s.scheduled) +} + +func (s *Simulated) WaitForTimers(n int) { + s.mu.Lock() + defer s.mu.Unlock() + s.init() + + for len(s.scheduled) < n { + s.cond.Wait() + } +} + +// Now implements Clock. +func (s *Simulated) Now() AbsTime { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.now +} + +// Sleep implements Clock. +func (s *Simulated) Sleep(d time.Duration) { + <-s.After(d) +} + +// After implements Clock. +func (s *Simulated) After(d time.Duration) <-chan time.Time { + after := make(chan time.Time, 1) + s.insert(d, func() { + after <- (time.Time{}).Add(time.Duration(s.now)) + }) + return after +} + +func (s *Simulated) insert(d time.Duration, do func()) { + s.mu.Lock() + defer s.mu.Unlock() + s.init() + + at := s.now + AbsTime(d) + l, h := 0, len(s.scheduled) + ll := h + for l != h { + m := (l + h) / 2 + if at < s.scheduled[m].at { + h = m + } else { + l = m + 1 + } + } + s.scheduled = append(s.scheduled, event{}) + copy(s.scheduled[l+1:], s.scheduled[l:ll]) + s.scheduled[l] = event{do: do, at: at} + s.cond.Broadcast() +} + +func (s *Simulated) init() { + if s.cond == nil { + s.cond = sync.NewCond(&s.mu) + } +} diff --git a/common/prque/prque.go b/common/prque/prque.go new file mode 100755 index 0000000000..9fd31a2e5d --- /dev/null +++ b/common/prque/prque.go @@ -0,0 +1,57 @@ +// This is a duplicated and slightly modified version of "gopkg.in/karalabe/cookiejar.v2/collections/prque". + +package prque + +import ( + "container/heap" +) + +// Priority queue data structure. +type Prque struct { + cont *sstack +} + +// Creates a new priority queue. +func New(setIndex setIndexCallback) *Prque { + return &Prque{newSstack(setIndex)} +} + +// Pushes a value with a given priority into the queue, expanding if necessary. +func (p *Prque) Push(data interface{}, priority int64) { + heap.Push(p.cont, &item{data, priority}) +} + +// Pops the value with the greates priority off the stack and returns it. +// Currently no shrinking is done. +func (p *Prque) Pop() (interface{}, int64) { + item := heap.Pop(p.cont).(*item) + return item.value, item.priority +} + +// Pops only the item from the queue, dropping the associated priority value. +func (p *Prque) PopItem() interface{} { + return heap.Pop(p.cont).(*item).value +} + +// Remove removes the element with the given index. +func (p *Prque) Remove(i int) interface{} { + if i < 0 { + return nil + } + return heap.Remove(p.cont, i) +} + +// Checks whether the priority queue is empty. +func (p *Prque) Empty() bool { + return p.cont.Len() == 0 +} + +// Returns the number of element in the priority queue. +func (p *Prque) Size() int { + return p.cont.Len() +} + +// Clears the contents of the priority queue. +func (p *Prque) Reset() { + *p = *New(p.cont.setIndex) +} diff --git a/common/prque/sstack.go b/common/prque/sstack.go new file mode 100755 index 0000000000..4875dae99d --- /dev/null +++ b/common/prque/sstack.go @@ -0,0 +1,106 @@ +// This is a duplicated and slightly modified version of "gopkg.in/karalabe/cookiejar.v2/collections/prque". + +package prque + +// The size of a block of data +const blockSize = 4096 + +// A prioritized item in the sorted stack. +// +// Note: priorities can "wrap around" the int64 range, a comes before b if (a.priority - b.priority) > 0. +// The difference between the lowest and highest priorities in the queue at any point should be less than 2^63. +type item struct { + value interface{} + priority int64 +} + +// setIndexCallback is called when the element is moved to a new index. +// Providing setIndexCallback is optional, it is needed only if the application needs +// to delete elements other than the top one. +type setIndexCallback func(a interface{}, i int) + +// Internal sortable stack data structure. Implements the Push and Pop ops for +// the stack (heap) functionality and the Len, Less and Swap methods for the +// sortability requirements of the heaps. +type sstack struct { + setIndex setIndexCallback + size int + capacity int + offset int + + blocks [][]*item + active []*item +} + +// Creates a new, empty stack. +func newSstack(setIndex setIndexCallback) *sstack { + result := new(sstack) + result.setIndex = setIndex + result.active = make([]*item, blockSize) + result.blocks = [][]*item{result.active} + result.capacity = blockSize + return result +} + +// Pushes a value onto the stack, expanding it if necessary. Required by +// heap.Interface. +func (s *sstack) Push(data interface{}) { + if s.size == s.capacity { + s.active = make([]*item, blockSize) + s.blocks = append(s.blocks, s.active) + s.capacity += blockSize + s.offset = 0 + } else if s.offset == blockSize { + s.active = s.blocks[s.size/blockSize] + s.offset = 0 + } + if s.setIndex != nil { + s.setIndex(data.(*item).value, s.size) + } + s.active[s.offset] = data.(*item) + s.offset++ + s.size++ +} + +// Pops a value off the stack and returns it. Currently no shrinking is done. +// Required by heap.Interface. +func (s *sstack) Pop() (res interface{}) { + s.size-- + s.offset-- + if s.offset < 0 { + s.offset = blockSize - 1 + s.active = s.blocks[s.size/blockSize] + } + res, s.active[s.offset] = s.active[s.offset], nil + if s.setIndex != nil { + s.setIndex(res.(*item).value, -1) + } + return +} + +// Returns the length of the stack. Required by sort.Interface. +func (s *sstack) Len() int { + return s.size +} + +// Compares the priority of two elements of the stack (higher is first). +// Required by sort.Interface. +func (s *sstack) Less(i, j int) bool { + return (s.blocks[i/blockSize][i%blockSize].priority - s.blocks[j/blockSize][j%blockSize].priority) > 0 +} + +// Swaps two elements in the stack. Required by sort.Interface. +func (s *sstack) Swap(i, j int) { + ib, io, jb, jo := i/blockSize, i%blockSize, j/blockSize, j%blockSize + a, b := s.blocks[jb][jo], s.blocks[ib][io] + if s.setIndex != nil { + s.setIndex(a.value, i) + s.setIndex(b.value, j) + } + s.blocks[ib][io], s.blocks[jb][jo] = a, b +} + +// Resets the stack, effectively clearing its contents. +func (s *sstack) Reset() { + *s = *newSstack(s.setIndex) +} diff --git a/les/freeclient.go b/les/freeclient.go new file mode 100644 index 0000000000..5ee607be8f --- /dev/null +++ b/les/freeclient.go @@ -0,0 +1,278 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package les implements the Light Ethereum Subprotocol. +package les + +import ( + "io" + "math" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/common/prque" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +// freeClientPool implements a client database that limits the connection time +// of each client and manages accepting/rejecting incoming connections and even +// kicking out some connected clients. The pool calculates recent usage time +// for each known client (a value that increases linearly when the client is +// connected and decreases exponentially when not connected). Clients with lower +// recent usage are preferred, unknown nodes have the highest priority. Already +// connected nodes receive a small bias in their favor in order to avoid accepting +// and instantly kicking out clients. +// +// Note: the pool can use any string for client identification. Using signature +// keys for that purpose would not make sense when being known has a negative +// value for the client. Currently the LES protocol manager uses IP addresses +// (without port address) to identify clients. +type freeClientPool struct { + db ethdb.Database + lock sync.Mutex + clock mclock.Clock + closed bool + + connectedLimit, totalLimit int + + addressMap map[string]*freeClientPoolEntry + connPool, disconnPool *prque.Prque + startupTime mclock.AbsTime + logOffsetAtStartup int64 +} + +const ( + recentUsageExpTC = time.Hour // time constant of the exponential weighting window for "recent" server usage + fixedPointMultiplier = 0x1000000 // constant to convert logarithms to fixed point format + connectedBias = time.Minute // this bias is applied in favor of already connected clients in order to avoid kicking them out very soon +) + +// newFreeClientPool creates a new free client pool +func newFreeClientPool(db ethdb.Database, connectedLimit, totalLimit int, clock mclock.Clock) *freeClientPool { + pool := &freeClientPool{ + db: db, + clock: clock, + addressMap: make(map[string]*freeClientPoolEntry), + connPool: prque.New(poolSetIndex), + disconnPool: prque.New(poolSetIndex), + connectedLimit: connectedLimit, + totalLimit: totalLimit, + } + pool.loadFromDb() + return pool +} + +func (f *freeClientPool) stop() { + f.lock.Lock() + f.closed = true + f.saveToDb() + f.lock.Unlock() +} + +// connect should be called after a successful handshake. If the connection was +// rejected, there is no need to call disconnect. +// +// Note: the disconnectFn callback should not block. +func (f *freeClientPool) connect(address string, disconnectFn func()) bool { + f.lock.Lock() + defer f.lock.Unlock() + + if f.closed { + return false + } + e := f.addressMap[address] + now := f.clock.Now() + var recentUsage int64 + if e == nil { + e = &freeClientPoolEntry{address: address, index: -1} + f.addressMap[address] = e + } else { + if e.connected { + log.Debug("Client already connected", "address", address) + return false + } + recentUsage = int64(math.Exp(float64(e.logUsage-f.logOffset(now)) / fixedPointMultiplier)) + } + e.linUsage = recentUsage - int64(now) + // check whether (linUsage+connectedBias) is smaller than the highest entry in the connected pool + if f.connPool.Size() == f.connectedLimit { + i := f.connPool.PopItem().(*freeClientPoolEntry) + if e.linUsage+int64(connectedBias)-i.linUsage < 0 { + // kick it out and accept the new client + f.connPool.Remove(i.index) + f.calcLogUsage(i, now) + i.connected = false + f.disconnPool.Push(i, -i.logUsage) + log.Debug("Client kicked out", "address", i.address) + i.disconnectFn() + } else { + // keep the old client and reject the new one + f.connPool.Push(i, i.linUsage) + log.Debug("Client rejected", "address", address) + return false + } + } + f.disconnPool.Remove(e.index) + e.connected = true + e.disconnectFn = disconnectFn + f.connPool.Push(e, e.linUsage) + if f.connPool.Size()+f.disconnPool.Size() > f.totalLimit { + f.disconnPool.Pop() + } + log.Debug("Client accepted", "address", address) + return true +} + +// disconnect should be called when a connection is terminated. If the disconnection +// was initiated by the pool itself using disconnectFn then calling disconnect is +// not necessary but permitted. +func (f *freeClientPool) disconnect(address string) { + f.lock.Lock() + defer f.lock.Unlock() + + if f.closed { + return + } + e := f.addressMap[address] + now := f.clock.Now() + if !e.connected { + log.Debug("Client already disconnected", "address", address) + return + } + + f.connPool.Remove(e.index) + f.calcLogUsage(e, now) + e.connected = false + f.disconnPool.Push(e, -e.logUsage) + log.Debug("Client disconnected", "address", address) +} + +// logOffset calculates the time-dependent offset for the logarithmic +// representation of recent usage +func (f *freeClientPool) logOffset(now mclock.AbsTime) int64 { + // Note: fixedPointMultiplier acts as a multiplier here; the reason for dividing the divisor + // is to avoid int64 overflow. We assume that int64(recentUsageExpTC) >> fixedPointMultiplier. + logDecay := int64((time.Duration(now - f.startupTime)) / (recentUsageExpTC / fixedPointMultiplier)) + return f.logOffsetAtStartup + logDecay +} + +// calcLogUsage converts recent usage from linear to logarithmic representation +// when disconnecting a peer or closing the client pool +func (f *freeClientPool) calcLogUsage(e *freeClientPoolEntry, now mclock.AbsTime) { + dt := e.linUsage + int64(now) + if dt < 1 { + dt = 1 + } + e.logUsage = int64(math.Log(float64(dt))*fixedPointMultiplier) + f.logOffset(now) +} + +// freeClientPoolStorage is the RLP representation of the pool's database storage +type freeClientPoolStorage struct { + LogOffset uint64 + List []*freeClientPoolEntry +} + +// loadFromDb restores pool status from the database storage +// (automatically called at initialization) +func (f *freeClientPool) loadFromDb() { + enc, err := f.db.Get([]byte("freeClientPool")) + if err != nil { + return + } + var storage freeClientPoolStorage + err = rlp.DecodeBytes(enc, &storage) + if err != nil { + log.Error("Failed to decode client list", "err", err) + return + } + f.logOffsetAtStartup = int64(storage.LogOffset) + f.startupTime = f.clock.Now() + for _, e := range storage.List { + log.Debug("Loaded free client record", "address", e.address, "logUsage", e.logUsage) + f.addressMap[e.address] = e + f.disconnPool.Push(e, -e.logUsage) + } +} + +// saveToDb saves pool status to the database storage +// (automatically called during shutdown) +func (f *freeClientPool) saveToDb() { + now := f.clock.Now() + storage := freeClientPoolStorage{ + LogOffset: uint64(f.logOffset(now)), + List: make([]*freeClientPoolEntry, len(f.addressMap)), + } + i := 0 + for _, e := range f.addressMap { + if e.connected { + f.calcLogUsage(e, now) + } + storage.List[i] = e + i++ + } + enc, err := rlp.EncodeToBytes(storage) + if err != nil { + log.Error("Failed to encode client list", "err", err) + } else { + f.db.Put([]byte("freeClientPool"), enc) + } +} + +// freeClientPoolEntry represents a client address known by the pool. +// When connected, recent usage is calculated as linUsage + int64(clock.Now()) +// When disconnected, it is calculated as exp(logUsage - logOffset) where logOffset +// also grows linearly with time while the server is running. +// Conversion between linear and logarithmic representation happens when connecting +// or disconnecting the node. +// +// Note: linUsage and logUsage are values used with constantly growing offsets so +// even though they are close to each other at any time they may wrap around int64 +// limits over time. Comparison should be performed accordingly. +type freeClientPoolEntry struct { + address string + connected bool + disconnectFn func() + linUsage, logUsage int64 + index int +} + +func (e *freeClientPoolEntry) EncodeRLP(w io.Writer) error { + return rlp.Encode(w, []interface{}{e.address, uint64(e.logUsage)}) +} + +func (e *freeClientPoolEntry) DecodeRLP(s *rlp.Stream) error { + var entry struct { + Address string + LogUsage uint64 + } + if err := s.Decode(&entry); err != nil { + return err + } + e.address = entry.Address + e.logUsage = int64(entry.LogUsage) + e.connected = false + e.index = -1 + return nil +} + +// poolSetIndex callback is used by both priority queues to set/update the index of +// the element in the queue. Index is needed to remove elements other than the top one. +func poolSetIndex(a interface{}, i int) { + a.(*freeClientPoolEntry).index = i +} diff --git a/les/freeclient_test.go b/les/freeclient_test.go new file mode 100644 index 0000000000..e95abc7aad --- /dev/null +++ b/les/freeclient_test.go @@ -0,0 +1,139 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package light implements on-demand retrieval capable state and chain objects +// for the Ethereum Light Client. +package les + +import ( + "fmt" + "math/rand" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/ethdb" +) + +func TestFreeClientPoolL10C100(t *testing.T) { + testFreeClientPool(t, 10, 100) +} + +func TestFreeClientPoolL40C200(t *testing.T) { + testFreeClientPool(t, 40, 200) +} + +func TestFreeClientPoolL100C300(t *testing.T) { + testFreeClientPool(t, 100, 300) +} + +const testFreeClientPoolTicks = 500000 + +func testFreeClientPool(t *testing.T, connLimit, clientCount int) { + var ( + clock mclock.Simulated + db = ethdb.NewMemDatabase() + pool = newFreeClientPool(db, connLimit, 10000, &clock) + connected = make([]bool, clientCount) + connTicks = make([]int, clientCount) + disconnCh = make(chan int, clientCount) + ) + peerId := func(i int) string { + return fmt.Sprintf("test peer #%d", i) + } + disconnFn := func(i int) func() { + return func() { + disconnCh <- i + } + } + + // pool should accept new peers up to its connected limit + for i := 0; i < connLimit; i++ { + if pool.connect(peerId(i), disconnFn(i)) { + connected[i] = true + } else { + t.Fatalf("Test peer #%d rejected", i) + } + } + // since all accepted peers are new and should not be kicked out, the next one should be rejected + if pool.connect(peerId(connLimit), disconnFn(connLimit)) { + connected[connLimit] = true + t.Fatalf("Peer accepted over connected limit") + } + + // randomly connect and disconnect peers, expect to have a similar total connection time at the end + for tickCounter := 0; tickCounter < testFreeClientPoolTicks; tickCounter++ { + clock.Run(1 * time.Second) + + i := rand.Intn(clientCount) + if connected[i] { + pool.disconnect(peerId(i)) + connected[i] = false + connTicks[i] += tickCounter + } else { + if pool.connect(peerId(i), disconnFn(i)) { + connected[i] = true + connTicks[i] -= tickCounter + } + } + pollDisconnects: + for { + select { + case i := <-disconnCh: + pool.disconnect(peerId(i)) + if connected[i] { + connTicks[i] += tickCounter + connected[i] = false + } + default: + break pollDisconnects + } + } + } + + expTicks := testFreeClientPoolTicks * connLimit / clientCount + expMin := expTicks - expTicks/10 + expMax := expTicks + expTicks/10 + + // check if the total connected time of peers are all in the expected range + for i, c := range connected { + if c { + connTicks[i] += testFreeClientPoolTicks + } + if connTicks[i] < expMin || connTicks[i] > expMax { + t.Errorf("Total connected time of test node #%d (%d) outside expected range (%d to %d)", i, connTicks[i], expMin, expMax) + } + } + + // a previously unknown peer should be accepted now + if !pool.connect("newPeer", func() {}) { + t.Fatalf("Previously unknown peer rejected") + } + + // close and restart pool + pool.stop() + pool = newFreeClientPool(db, connLimit, 10000, &clock) + + // try connecting all known peers (connLimit should be filled up) + for i := 0; i < clientCount; i++ { + pool.connect(peerId(i), func() {}) + } + // expect pool to remember known nodes and kick out one of them to accept a new one + if !pool.connect("newPeer2", func() {}) { + t.Errorf("Previously unknown peer rejected after restarting pool") + } + pool.stop() +} diff --git a/les/handler.go b/les/handler.go index 2fc4cde349..91a235bf0a 100644 --- a/les/handler.go +++ b/les/handler.go @@ -28,6 +28,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -104,6 +105,7 @@ type ProtocolManager struct { odr *LesOdr server *LesServer serverPool *serverPool + clientPool *freeClientPool lesTopic discv5.Topic reqDist *requestDistributor retriever *retrieveManager @@ -226,6 +228,7 @@ func (pm *ProtocolManager) Start(maxPeers int) { if pm.lightSync { go pm.syncer() } else { + pm.clientPool = newFreeClientPool(pm.chainDb, maxPeers, 10000, mclock.System{}) go func() { for range pm.newPeerCh { } @@ -243,6 +246,9 @@ func (pm *ProtocolManager) Stop() { pm.noMorePeers <- struct{}{} close(pm.quitSync) // quits syncer, fetcher + if pm.clientPool != nil { + pm.clientPool.stop() + } // Disconnect existing sessions. // This also closes the gate for any new registrations on the peer set. @@ -264,7 +270,8 @@ func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgRea // this function terminates, the peer is disconnected. func (pm *ProtocolManager) handle(p *peer) error { // Ignore maxPeers if this is a trusted peer - if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted { + // In server mode we try to check into the client pool after handshake + if pm.lightSync && pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted { return p2p.DiscTooManyPeers } @@ -282,6 +289,19 @@ func (pm *ProtocolManager) handle(p *peer) error { p.Log().Debug("Light Ethereum handshake failed", "err", err) return err } + + if !pm.lightSync && !p.Peer.Info().Network.Trusted { + addr, ok := p.RemoteAddr().(*net.TCPAddr) + // test peer address is not a tcp address, don't use client pool if can not typecast + if ok { + id := addr.IP.String() + if !pm.clientPool.connect(id, func() { go pm.removePeer(p.id) }) { + return p2p.DiscTooManyPeers + } + defer pm.clientPool.disconnect(id) + } + } + if rw, ok := p.rw.(*meteredMsgReadWriter); ok { rw.Init(p.version) } From b52bb31b767a3e31266e6138e24f7226ebbd8882 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 14 Aug 2018 22:59:18 +0200 Subject: [PATCH 08/96] p2p/discv5: add delay to refresh cycle when no seed nodes are found (#16994) --- p2p/discv5/net.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index 4c39c05533..b93c93d648 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -678,7 +678,7 @@ func (net *Network) refresh(done chan<- struct{}) { } if len(seeds) == 0 { log.Trace("no seed nodes found") - close(done) + time.AfterFunc(time.Second*10, func() { close(done) }) return } for _, n := range seeds { From 212bba47ff13812ddabb642da463e58cda4ff20f Mon Sep 17 00:00:00 2001 From: Jeff Prestes Date: Wed, 15 Aug 2018 04:15:42 -0300 Subject: [PATCH 09/96] backends: configurable gas limit to allow testing large contracts (#17358) * backends: increase gaslimit in order to allow tests of large contracts * backends: increase gaslimit in order to allow tests of large contracts * backends: increase gaslimit in order to allow tests of large contracts --- accounts/abi/bind/backends/simulated.go | 4 ++-- accounts/abi/bind/bind_test.go | 24 ++++++++++++------------ accounts/abi/bind/util_test.go | 8 +++++--- contracts/chequebook/cheque_test.go | 2 +- contracts/chequebook/gencode.go | 2 +- contracts/ens/ens_test.go | 2 +- 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index fa8828f61b..1d14f8c6f1 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -65,9 +65,9 @@ type SimulatedBackend struct { // NewSimulatedBackend creates a new binding backend using a simulated blockchain // for testing purposes. -func NewSimulatedBackend(alloc core.GenesisAlloc) *SimulatedBackend { +func NewSimulatedBackend(alloc core.GenesisAlloc, gasLimit uint64) *SimulatedBackend { database := ethdb.NewMemDatabase() - genesis := core.Genesis{Config: params.AllEthashProtocolChanges, Alloc: alloc} + genesis := core.Genesis{Config: params.AllEthashProtocolChanges, GasLimit: gasLimit, Alloc: alloc} genesis.MustCommit(database) blockchain, _ := core.NewBlockChain(database, nil, genesis.Config, ethash.NewFaker(), vm.Config{}) diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 2a5a886487..0e5b1c1616 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -229,7 +229,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy an interaction tester contract and call a transaction on it _, _, interactor, err := DeployInteractor(auth, sim, "Deploy string") @@ -270,7 +270,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a tuple tester contract and execute a structured call on it _, _, getter, err := DeployGetter(auth, sim) @@ -302,7 +302,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a tuple tester contract and execute a structured call on it _, _, tupler, err := DeployTupler(auth, sim) @@ -344,7 +344,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a slice tester contract and execute a n array call on it _, _, slicer, err := DeploySlicer(auth, sim) @@ -378,7 +378,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a default method invoker contract and execute its default method _, _, defaulter, err := DeployDefaulter(auth, sim) @@ -411,7 +411,7 @@ var bindTests = []struct { `[{"constant":true,"inputs":[],"name":"String","outputs":[{"name":"","type":"string"}],"type":"function"}]`, ` // Create a simulator and wrap a non-deployed contract - sim := backends.NewSimulatedBackend(nil) + sim := backends.NewSimulatedBackend(nil, uint64(10000000000)) nonexistent, err := NewNonExistent(common.Address{}, sim) if err != nil { @@ -447,7 +447,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a funky gas pattern contract _, _, limiter, err := DeployFunkyGasPattern(auth, sim) @@ -482,7 +482,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a sender tester contract and execute a structured call on it _, _, callfrom, err := DeployCallFrom(auth, sim) @@ -542,7 +542,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy a underscorer tester contract and execute a structured call on it _, _, underscorer, err := DeployUnderscorer(auth, sim) @@ -612,7 +612,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) // Deploy an eventer contract _, _, eventer, err := DeployEventer(auth, sim) @@ -761,7 +761,7 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth := bind.NewKeyedTransactor(key) - sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}) + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) //deploy the test contract _, _, testContract, err := DeployDeeplyNestedArray(auth, sim) @@ -820,7 +820,7 @@ func TestBindings(t *testing.T) { t.Skip("go sdk not found for testing") } // Skip the test if the go-ethereum sources are symlinked (https://github.com/golang/go/issues/14845) - linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil))\n}") + linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil,uint64(10000000000)))\n}") linkTestDeps, err := imports.Process(os.TempDir(), []byte(linkTestCode), nil) if err != nil { t.Fatalf("failed check for goimports symlink bug: %v", err) diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index 49e6dc813a..8f4092971f 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -53,9 +53,11 @@ var waitDeployedTests = map[string]struct { func TestWaitDeployed(t *testing.T) { for name, test := range waitDeployedTests { - backend := backends.NewSimulatedBackend(core.GenesisAlloc{ - crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)}, - }) + backend := backends.NewSimulatedBackend( + core.GenesisAlloc{ + crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) // Create the transaction. tx := types.NewContractCreation(0, big.NewInt(0), test.gas, big.NewInt(1), common.FromHex(test.code)) diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go index 6b6b28e657..4bd2e176b1 100644 --- a/contracts/chequebook/cheque_test.go +++ b/contracts/chequebook/cheque_test.go @@ -46,7 +46,7 @@ func newTestBackend() *backends.SimulatedBackend { addr0: {Balance: big.NewInt(1000000000)}, addr1: {Balance: big.NewInt(1000000000)}, addr2: {Balance: big.NewInt(1000000000)}, - }) + }, 10000000) } func deploy(prvKey *ecdsa.PrivateKey, amount *big.Int, backend *backends.SimulatedBackend) (common.Address, error) { diff --git a/contracts/chequebook/gencode.go b/contracts/chequebook/gencode.go index 45f6d68f3e..ddfe8d1512 100644 --- a/contracts/chequebook/gencode.go +++ b/contracts/chequebook/gencode.go @@ -40,7 +40,7 @@ var ( ) func main() { - backend := backends.NewSimulatedBackend(testAlloc) + backend := backends.NewSimulatedBackend(testAlloc, uint64(100000000)) auth := bind.NewKeyedTransactor(testKey) // Deploy the contract, get the code. diff --git a/contracts/ens/ens_test.go b/contracts/ens/ens_test.go index 6ad8447082..411b04197c 100644 --- a/contracts/ens/ens_test.go +++ b/contracts/ens/ens_test.go @@ -35,7 +35,7 @@ var ( ) func TestENS(t *testing.T) { - contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}) + contractBackend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: big.NewInt(1000000000)}}, 10000000) transactOpts := bind.NewKeyedTransactor(key) ensAddr, ens, err := DeployENS(transactOpts, contractBackend) From 2a17fe25612b57d943862459dba88666685ffd69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 15 Aug 2018 11:01:49 +0300 Subject: [PATCH 10/96] cmd: polish miner flags, deprecate olds, add upgrade path --- cmd/geth/chaincmd.go | 12 ++--- cmd/geth/main.go | 29 ++++++---- cmd/geth/usage.go | 15 +++--- cmd/puppeth/module_node.go | 2 +- cmd/utils/flags.go | 108 +++++++++++++++++++++++-------------- 5 files changed, 104 insertions(+), 62 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index ff27a9dfb4..87548865be 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -48,7 +48,6 @@ var ( ArgsUsage: "", Flags: []cli.Flag{ utils.DataDirFlag, - utils.LightModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -66,7 +65,7 @@ It expects the genesis file as argument.`, Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.LightModeFlag, + utils.SyncModeFlag, utils.GCModeFlag, utils.CacheDatabaseFlag, utils.CacheGCFlag, @@ -87,7 +86,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`, Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.LightModeFlag, + utils.SyncModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -105,7 +104,7 @@ be gzipped.`, Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.LightModeFlag, + utils.SyncModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -119,7 +118,7 @@ be gzipped.`, Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.LightModeFlag, + utils.SyncModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -149,7 +148,6 @@ The first argument must be the directory containing the blockchain to download f ArgsUsage: " ", Flags: []cli.Flag{ utils.DataDirFlag, - utils.LightModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -163,7 +161,7 @@ Remove blockchain and state databases`, Flags: []cli.Flag{ utils.DataDirFlag, utils.CacheFlag, - utils.LightModeFlag, + utils.SyncModeFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` diff --git a/cmd/geth/main.go b/cmd/geth/main.go index d556ad92c3..a063860513 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -82,8 +82,6 @@ var ( utils.TxPoolAccountQueueFlag, utils.TxPoolGlobalQueueFlag, utils.TxPoolLifetimeFlag, - utils.FastSyncFlag, - utils.LightModeFlag, utils.SyncModeFlag, utils.GCModeFlag, utils.LightServFlag, @@ -96,12 +94,18 @@ var ( utils.ListenPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, - utils.EtherbaseFlag, - utils.GasPriceFlag, utils.MiningEnabledFlag, utils.MinerThreadsFlag, + utils.MinerLegacyThreadsFlag, utils.MinerNotifyFlag, - utils.TargetGasLimitFlag, + utils.MinerGasTargetFlag, + utils.MinerLegacyGasTargetFlag, + utils.MinerGasPriceFlag, + utils.MinerLegacyGasPriceFlag, + utils.MinerEtherbaseFlag, + utils.MinerLegacyEtherbaseFlag, + utils.MinerExtraDataFlag, + utils.MinerLegacyExtraDataFlag, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, @@ -122,7 +126,6 @@ var ( utils.NoCompactionFlag, utils.GpoBlocksFlag, utils.GpoPercentileFlag, - utils.ExtraDataFlag, configFileFlag, } @@ -324,7 +327,7 @@ func startNode(ctx *cli.Context, stack *node.Node) { // Start auxiliary services if enabled if ctx.GlobalBool(utils.MiningEnabledFlag.Name) || ctx.GlobalBool(utils.DeveloperFlag.Name) { // Mining only makes sense if a full Ethereum node is running - if ctx.GlobalBool(utils.LightModeFlag.Name) || ctx.GlobalString(utils.SyncModeFlag.Name) == "light" { + if ctx.GlobalString(utils.SyncModeFlag.Name) == "light" { utils.Fatalf("Light clients do not support mining") } var ethereum *eth.Ethereum @@ -332,7 +335,11 @@ func startNode(ctx *cli.Context, stack *node.Node) { utils.Fatalf("Ethereum service not running: %v", err) } // Use a reduced number of threads if requested - if threads := ctx.GlobalInt(utils.MinerThreadsFlag.Name); threads > 0 { + threads := ctx.GlobalInt(utils.MinerLegacyThreadsFlag.Name) + if ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) { + threads = ctx.GlobalInt(utils.MinerThreadsFlag.Name) + } + if threads > 0 { type threaded interface { SetThreads(threads int) } @@ -341,7 +348,11 @@ func startNode(ctx *cli.Context, stack *node.Node) { } } // Set the gas price to the limits from the CLI and start mining - ethereum.TxPool().SetGasPrice(utils.GlobalBig(ctx, utils.GasPriceFlag.Name)) + gasprice := utils.GlobalBig(ctx, utils.MinerLegacyGasPriceFlag.Name) + if ctx.IsSet(utils.MinerGasPriceFlag.Name) { + gasprice = utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) + } + ethereum.TxPool().SetGasPrice(gasprice) if err := ethereum.StartMining(true); err != nil { utils.Fatalf("Failed to start mining: %v", err) } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 9d63c68f7f..9e18f70472 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -186,10 +186,10 @@ var AppHelpFlagGroups = []flagGroup{ utils.MiningEnabledFlag, utils.MinerThreadsFlag, utils.MinerNotifyFlag, - utils.EtherbaseFlag, - utils.TargetGasLimitFlag, - utils.GasPriceFlag, - utils.ExtraDataFlag, + utils.MinerGasPriceFlag, + utils.MinerGasTargetFlag, + utils.MinerEtherbaseFlag, + utils.MinerExtraDataFlag, }, }, { @@ -231,8 +231,11 @@ var AppHelpFlagGroups = []flagGroup{ { Name: "DEPRECATED", Flags: []cli.Flag{ - utils.FastSyncFlag, - utils.LightModeFlag, + utils.MinerLegacyThreadsFlag, + utils.MinerLegacyGasTargetFlag, + utils.MinerLegacyGasPriceFlag, + utils.MinerLegacyEtherbaseFlag, + utils.MinerLegacyExtraDataFlag, }, }, { diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index a480a894eb..8ad41555e1 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -42,7 +42,7 @@ ADD genesis.json /genesis.json RUN \ echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh + echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d6142f246c..e3a8cc2eac 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -157,14 +157,6 @@ var ( Usage: "Document Root for HTTPClient file scheme", Value: DirectoryString{homeDir()}, } - FastSyncFlag = cli.BoolFlag{ - Name: "fast", - Usage: "Enable fast syncing through state downloads (replaced by --syncmode)", - } - LightModeFlag = cli.BoolFlag{ - Name: "light", - Usage: "Enable light client mode (replaced by --syncmode)", - } defaultSyncMode = eth.DefaultConfig.SyncMode SyncModeFlag = TextMarshalerFlag{ Name: "syncmode", @@ -321,29 +313,53 @@ var ( Usage: "Number of CPU threads to use for mining", Value: 0, } + MinerLegacyThreadsFlag = cli.IntFlag{ + Name: "minerthreads", + Usage: "Number of CPU threads to use for mining (deprecated, use --miner.threads)", + Value: 0, + } MinerNotifyFlag = cli.StringFlag{ Name: "miner.notify", Usage: "Comma separated HTTP URL list to notify of new work packages", } - TargetGasLimitFlag = cli.Uint64Flag{ - Name: "targetgaslimit", - Usage: "Target gas limit sets the artificial target gas floor for the blocks to mine", + MinerGasTargetFlag = cli.Uint64Flag{ + Name: "miner.gastarget", + Usage: "Target gas floor for mined blocks", Value: params.GenesisGasLimit, } - EtherbaseFlag = cli.StringFlag{ - Name: "etherbase", - Usage: "Public address for block mining rewards (default = first account created)", - Value: "0", + MinerLegacyGasTargetFlag = cli.Uint64Flag{ + Name: "targetgaslimit", + Usage: "Target gas floor for mined blocks (deprecated, use --miner.gastarget)", + Value: params.GenesisGasLimit, } - GasPriceFlag = BigFlag{ - Name: "gasprice", - Usage: "Minimal gas price to accept for mining a transactions", + MinerGasPriceFlag = BigFlag{ + Name: "miner.gasprice", + Usage: "Minimal gas price for mining a transactions", Value: eth.DefaultConfig.GasPrice, } - ExtraDataFlag = cli.StringFlag{ - Name: "extradata", + MinerLegacyGasPriceFlag = BigFlag{ + Name: "gasprice", + Usage: "Minimal gas price for mining a transactions (deprecated, use --miner.gasprice)", + Value: eth.DefaultConfig.GasPrice, + } + MinerEtherbaseFlag = cli.StringFlag{ + Name: "miner.etherbase", + Usage: "Public address for block mining rewards (default = first account)", + Value: "0", + } + MinerLegacyEtherbaseFlag = cli.StringFlag{ + Name: "etherbase", + Usage: "Public address for block mining rewards (default = first account, deprecated, use --miner.etherbase)", + Value: "0", + } + MinerExtraDataFlag = cli.StringFlag{ + Name: "miner.extradata", Usage: "Block extra data set by the miner (default = client version)", } + MinerLegacyExtraDataFlag = cli.StringFlag{ + Name: "extradata", + Usage: "Block extra data set by the miner (default = client version, deprecated, use --miner.extradata)", + } // Account settings UnlockedAccountFlag = cli.StringFlag{ Name: "unlock", @@ -813,10 +829,19 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error // setEtherbase retrieves the etherbase either from the directly specified // command line flags or from the keystore if CLI indexed. func setEtherbase(ctx *cli.Context, ks *keystore.KeyStore, cfg *eth.Config) { - if ctx.GlobalIsSet(EtherbaseFlag.Name) { - account, err := MakeAddress(ks, ctx.GlobalString(EtherbaseFlag.Name)) + // Extract the current etherbase, new flag overriding legacy one + var etherbase string + if ctx.GlobalIsSet(MinerLegacyEtherbaseFlag.Name) { + etherbase = ctx.GlobalString(MinerLegacyEtherbaseFlag.Name) + } + if ctx.GlobalIsSet(MinerEtherbaseFlag.Name) { + etherbase = ctx.GlobalString(MinerEtherbaseFlag.Name) + } + // Convert the etherbase into an address and configure it + if etherbase != "" { + account, err := MakeAddress(ks, etherbase) if err != nil { - Fatalf("Option %q: %v", EtherbaseFlag.Name, err) + Fatalf("Invalid miner etherbase: %v", err) } cfg.Etherbase = account.Address } @@ -847,7 +872,7 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { setBootstrapNodes(ctx, cfg) setBootstrapNodesV5(ctx, cfg) - lightClient := ctx.GlobalBool(LightModeFlag.Name) || ctx.GlobalString(SyncModeFlag.Name) == "light" + lightClient := ctx.GlobalString(SyncModeFlag.Name) == "light" lightServer := ctx.GlobalInt(LightServFlag.Name) != 0 lightPeers := ctx.GlobalInt(LightPeersFlag.Name) @@ -1052,8 +1077,6 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { // Avoid conflicting network flags checkExclusive(ctx, DeveloperFlag, TestnetFlag, RinkebyFlag) - checkExclusive(ctx, FastSyncFlag, LightModeFlag, SyncModeFlag) - checkExclusive(ctx, LightServFlag, LightModeFlag) checkExclusive(ctx, LightServFlag, SyncModeFlag, "light") ks := stack.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) @@ -1062,13 +1085,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) - switch { - case ctx.GlobalIsSet(SyncModeFlag.Name): + if ctx.GlobalIsSet(SyncModeFlag.Name) { cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) - case ctx.GlobalBool(FastSyncFlag.Name): - cfg.SyncMode = downloader.FastSync - case ctx.GlobalBool(LightModeFlag.Name): - cfg.SyncMode = downloader.LightSync } if ctx.GlobalIsSet(LightServFlag.Name) { cfg.LightServ = ctx.GlobalInt(LightServFlag.Name) @@ -1093,6 +1111,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } + if ctx.GlobalIsSet(MinerLegacyThreadsFlag.Name) { + cfg.MinerThreads = ctx.GlobalInt(MinerLegacyThreadsFlag.Name) + } if ctx.GlobalIsSet(MinerThreadsFlag.Name) { cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name) } @@ -1102,11 +1123,17 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(DocRootFlag.Name) { cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) } - if ctx.GlobalIsSet(ExtraDataFlag.Name) { - cfg.ExtraData = []byte(ctx.GlobalString(ExtraDataFlag.Name)) + if ctx.GlobalIsSet(MinerLegacyExtraDataFlag.Name) { + cfg.ExtraData = []byte(ctx.GlobalString(MinerLegacyExtraDataFlag.Name)) } - if ctx.GlobalIsSet(GasPriceFlag.Name) { - cfg.GasPrice = GlobalBig(ctx, GasPriceFlag.Name) + if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { + cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) + } + if ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { + cfg.GasPrice = GlobalBig(ctx, MinerLegacyGasPriceFlag.Name) + } + if ctx.GlobalIsSet(MinerGasPriceFlag.Name) { + cfg.GasPrice = GlobalBig(ctx, MinerGasPriceFlag.Name) } if ctx.GlobalIsSet(VMEnableDebugFlag.Name) { // TODO(fjl): force-enable this in --dev mode @@ -1148,7 +1175,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { log.Info("Using developer account", "address", developer.Address) cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.GlobalInt(DeveloperPeriodFlag.Name)), developer.Address) - if !ctx.GlobalIsSet(GasPriceFlag.Name) { + if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) && !ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { cfg.GasPrice = big.NewInt(1) } } @@ -1223,7 +1250,10 @@ func RegisterEthStatsService(stack *node.Node, url string) { // SetupNetwork configures the system for either the main net or some test network. func SetupNetwork(ctx *cli.Context) { // TODO(fjl): move target gas limit into config - params.TargetGasLimit = ctx.GlobalUint64(TargetGasLimitFlag.Name) + params.TargetGasLimit = ctx.GlobalUint64(MinerLegacyGasTargetFlag.Name) + if ctx.GlobalIsSet(MinerGasTargetFlag.Name) { + params.TargetGasLimit = ctx.GlobalUint64(MinerGasTargetFlag.Name) + } } func SetupMetrics(ctx *cli.Context) { @@ -1254,7 +1284,7 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { handles = makeDatabaseHandles() ) name := "chaindata" - if ctx.GlobalBool(LightModeFlag.Name) { + if ctx.GlobalString(SyncModeFlag.Name) == "light" { name = "lightchaindata" } chainDb, err := stack.OpenDatabase(name, cache, handles) From 040aa2bb101e5e602308b24812bfbf2451b21174 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 15 Aug 2018 19:09:17 +0800 Subject: [PATCH 11/96] miner: streaming uncle blocks (#17320) * miner: stream uncle block * miner: polish --- miner/worker.go | 105 ++++++++++++++++++++++++------------------- miner/worker_test.go | 76 +++++++++++++++++++++++++++---- 2 files changed, 128 insertions(+), 53 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 81a63c29a4..fae480c84c 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -213,8 +213,9 @@ type worker struct { running int32 // The indicator whether the consensus engine is running or not. // Test hooks - newTaskHook func(*task) // Method to call upon receiving a new sealing task - fullTaskInterval func() // Method to call before pushing the full sealing task + newTaskHook func(*task) // Method to call upon receiving a new sealing task + skipSealHook func(*task) bool // Method to decide whether skipping the sealing. + fullTaskHook func() // Method to call before pushing the full sealing task } func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux) *worker { @@ -329,8 +330,32 @@ func (w *worker) mainLoop() { w.commitNewWork() case ev := <-w.chainSideCh: + if _, exist := w.possibleUncles[ev.Block.Hash()]; exist { + continue + } // Add side block to possible uncle block set. w.possibleUncles[ev.Block.Hash()] = ev.Block + // If our mining block contains less than 2 uncle blocks, + // add the new uncle block if valid and regenerate a mining block. + if w.isRunning() && w.current != nil && w.current.uncles.Cardinality() < 2 { + start := time.Now() + if err := w.commitUncle(w.current, ev.Block.Header()); err == nil { + var uncles []*types.Header + w.current.uncles.Each(func(item interface{}) bool { + hash, ok := item.(common.Hash) + if !ok { + return false + } + uncle, exist := w.possibleUncles[hash] + if !exist { + return false + } + uncles = append(uncles, uncle.Header()) + return true + }) + w.commit(uncles, nil, true, start) + } + } case ev := <-w.txsCh: // Apply transactions to the pending state if we're not mining. @@ -378,6 +403,10 @@ func (w *worker) seal(t *task, stop <-chan struct{}) { res *task ) + if w.skipSealHook != nil && w.skipSealHook(t) { + return + } + if t.block, err = w.engine.Seal(w.chain, t.block, stop); t.block != nil { log.Info("Successfully sealed new block", "number", t.block.Number(), "hash", t.block.Hash(), "elapsed", common.PrettyDuration(time.Since(t.createdAt))) @@ -637,30 +666,9 @@ func (w *worker) commitNewWork() { delete(w.possibleUncles, hash) } - var ( - emptyBlock, fullBlock *types.Block - emptyState, fullState *state.StateDB - ) - // Create an empty block based on temporary copied state for sealing in advance without waiting block // execution finished. - emptyState = env.state.Copy() - if emptyBlock, err = w.engine.Finalize(w.chain, header, emptyState, nil, uncles, nil); err != nil { - log.Error("Failed to finalize block for temporary sealing", "err", err) - } else { - // Push empty work in advance without applying pending transaction. - // The reason is transactions execution can cost a lot and sealer need to - // take advantage of this part time. - if w.isRunning() { - select { - case w.taskCh <- &task{receipts: nil, state: emptyState, block: emptyBlock, createdAt: time.Now()}: - log.Info("Commit new empty mining work", "number", emptyBlock.Number(), "uncles", len(uncles)) - case <-w.exitCh: - log.Info("Worker has exited") - return - } - } - } + w.commit(uncles, nil, false, tstart) // Fill the block with all available pending transactions. pending, err := w.eth.TxPool().Pending() @@ -676,31 +684,38 @@ func (w *worker) commitNewWork() { txs := types.NewTransactionsByPriceAndNonce(w.current.signer, pending) env.commitTransactions(w.mux, txs, w.chain, w.coinbase) - // Create the full block to seal with the consensus engine - fullState = env.state.Copy() - if fullBlock, err = w.engine.Finalize(w.chain, header, fullState, env.txs, uncles, env.receipts); err != nil { - log.Error("Failed to finalize block for sealing", "err", err) - return - } - // Deep copy receipts here to avoid interaction between different tasks. - cpy := make([]*types.Receipt, len(env.receipts)) - for i, l := range env.receipts { - cpy[i] = new(types.Receipt) - *cpy[i] = *l - } - // We only care about logging if we're actually mining. - if w.isRunning() { - if w.fullTaskInterval != nil { - w.fullTaskInterval() - } + w.commit(uncles, w.fullTaskHook, true, tstart) +} +// commit runs any post-transaction state modifications, assembles the final block +// and commits new work if consensus engine is running. +func (w *worker) commit(uncles []*types.Header, interval func(), update bool, start time.Time) error { + // Deep copy receipts here to avoid interaction between different tasks. + receipts := make([]*types.Receipt, len(w.current.receipts)) + for i, l := range w.current.receipts { + receipts[i] = new(types.Receipt) + *receipts[i] = *l + } + s := w.current.state.Copy() + block, err := w.engine.Finalize(w.chain, w.current.header, s, w.current.txs, uncles, w.current.receipts) + if err != nil { + return err + } + if w.isRunning() { + if interval != nil { + interval() + } select { - case w.taskCh <- &task{receipts: cpy, state: fullState, block: fullBlock, createdAt: time.Now()}: - w.unconfirmed.Shift(fullBlock.NumberU64() - 1) - log.Info("Commit new full mining work", "number", fullBlock.Number(), "txs", env.tcount, "uncles", len(uncles), "elapsed", common.PrettyDuration(time.Since(tstart))) + case w.taskCh <- &task{receipts: receipts, state: s, block: block, createdAt: time.Now()}: + w.unconfirmed.Shift(block.NumberU64() - 1) + log.Info("Commit new mining work", "number", block.Number(), "txs", w.current.tcount, "uncles", len(uncles), + "elapsed", common.PrettyDuration(time.Since(start))) case <-w.exitCh: log.Info("Worker has exited") } } - w.updateSnapshot() + if update { + w.updateSnapshot() + } + return nil } diff --git a/miner/worker_test.go b/miner/worker_test.go index 5823a608ef..408c47e3b3 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -59,7 +59,7 @@ func init() { ethashChainConfig = params.TestChainConfig cliqueChainConfig = params.TestChainConfig cliqueChainConfig.Clique = ¶ms.CliqueConfig{ - Period: 1, + Period: 10, Epoch: 30000, } tx1, _ := types.SignTx(types.NewTransaction(0, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) @@ -74,6 +74,7 @@ type testWorkerBackend struct { txPool *core.TxPool chain *core.BlockChain testTxFeed event.Feed + uncleBlock *types.Block } func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) *testWorkerBackend { @@ -93,15 +94,19 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine default: t.Fatal("unexpect consensus engine type") } - gspec.MustCommit(db) + genesis := gspec.MustCommit(db) chain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) txpool := core.NewTxPool(testTxPoolConfig, chainConfig, chain) + blocks, _ := core.GenerateChain(chainConfig, genesis, engine, db, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(acc1Addr) + }) return &testWorkerBackend{ - db: db, - chain: chain, - txPool: txpool, + db: db, + chain: chain, + txPool: txpool, + uncleBlock: blocks[0], } } @@ -188,7 +193,7 @@ func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consens taskCh <- struct{}{} } } - w.fullTaskInterval = func() { + w.fullTaskHook = func() { time.Sleep(100 * time.Millisecond) } @@ -202,11 +207,66 @@ func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consens w.start() for i := 0; i < 2; i += 1 { - to := time.NewTimer(time.Second) select { case <-taskCh: - case <-to.C: + case <-time.NewTimer(time.Second).C: t.Error("new task timeout") } } } + +func TestStreamUncleBlock(t *testing.T) { + ethash := ethash.NewFaker() + defer ethash.Close() + + w, b := newTestWorker(t, ethashChainConfig, ethash) + defer w.close() + + var taskCh = make(chan struct{}) + + taskIndex := 0 + w.newTaskHook = func(task *task) { + if task.block.NumberU64() == 1 { + if taskIndex == 2 { + has := task.block.Header().UncleHash + want := types.CalcUncleHash([]*types.Header{b.uncleBlock.Header()}) + if has != want { + t.Errorf("uncle hash mismatch, has %s, want %s", has.Hex(), want.Hex()) + } + } + taskCh <- struct{}{} + taskIndex += 1 + } + } + w.skipSealHook = func(task *task) bool { + return true + } + w.fullTaskHook = func() { + time.Sleep(100 * time.Millisecond) + } + + // Ensure worker has finished initialization + for { + b := w.pendingBlock() + if b != nil && b.NumberU64() == 1 { + break + } + } + + w.start() + // Ignore the first two works + for i := 0; i < 2; i += 1 { + select { + case <-taskCh: + case <-time.NewTimer(time.Second).C: + t.Error("new task timeout") + } + } + b.PostChainEvents([]interface{}{core.ChainSideEvent{Block: b.uncleBlock}}) + + select { + case <-taskCh: + case <-time.NewTimer(time.Second).C: + t.Error("new task timeout") + } +} From d8541a9f99c58d97ba4908c3a768e518f28d2441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 15 Aug 2018 13:50:16 +0300 Subject: [PATCH 12/96] consensus/ethash: use DAGs for remote mining, generate async --- consensus/ethash/consensus.go | 52 +++++++++++++++++++++++++++-------- consensus/ethash/ethash.go | 44 +++++++++++++++++++++++------ consensus/ethash/sealer.go | 11 ++++---- 3 files changed, 83 insertions(+), 24 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index e18a06d52a..86fd997ae4 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -461,6 +461,13 @@ func calcDifficultyFrontier(time uint64, parent *types.Header) *big.Int { // VerifySeal implements consensus.Engine, checking whether the given block satisfies // the PoW difficulty requirements. func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Header) error { + return ethash.verifySeal(chain, header, false) +} + +// verifySeal checks whether a block satisfies the PoW difficulty requirements, +// either using the usual ethash cache for it, or alternatively using a full DAG +// to make remote mining fast. +func (ethash *Ethash) verifySeal(chain consensus.ChainReader, header *types.Header, fulldag bool) error { // If we're running a fake PoW, accept any seal as valid if ethash.config.PowMode == ModeFake || ethash.config.PowMode == ModeFullFake { time.Sleep(ethash.fakeDelay) @@ -471,25 +478,48 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head } // If we're running a shared PoW, delegate verification to it if ethash.shared != nil { - return ethash.shared.VerifySeal(chain, header) + return ethash.shared.verifySeal(chain, header, fulldag) } // Ensure that we have a valid difficulty for the block if header.Difficulty.Sign() <= 0 { return errInvalidDifficulty } - // Recompute the digest and PoW value and verify against the header + // Recompute the digest and PoW values number := header.Number.Uint64() - cache := ethash.cache(number) - size := datasetSize(number) - if ethash.config.PowMode == ModeTest { - size = 32 * 1024 - } - digest, result := hashimotoLight(size, cache.cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) - // Caches are unmapped in a finalizer. Ensure that the cache stays live - // until after the call to hashimotoLight so it's not unmapped while being used. - runtime.KeepAlive(cache) + var ( + digest []byte + result []byte + ) + // If fast-but-heavy PoW verification was requested, use an ethash dataset + if fulldag { + dataset := ethash.dataset(number, true) + if dataset.generated() { + digest, result = hashimotoFull(dataset.dataset, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + // Datasets are unmapped in a finalizer. Ensure that the dataset stays alive + // until after the call to hashimotoFull so it's not unmapped while being used. + runtime.KeepAlive(dataset) + } else { + // Dataset not yet generated, don't hang, use a cache instead + fulldag = false + } + } + // If slow-but-light PoW verification was requested (or DAG not yet ready), use an ethash cache + if !fulldag { + cache := ethash.cache(number) + + size := datasetSize(number) + if ethash.config.PowMode == ModeTest { + size = 32 * 1024 + } + digest, result = hashimotoLight(size, cache.cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + + // Caches are unmapped in a finalizer. Ensure that the cache stays alive + // until after the call to hashimotoLight so it's not unmapped while being used. + runtime.KeepAlive(cache) + } + // Verify the calculated values against the ones provided in the header if !bytes.Equal(header.MixDigest[:], digest) { return errInvalidMixDigest } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 19c94deb6b..d98c3371c5 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -29,6 +29,7 @@ import ( "runtime" "strconv" "sync" + "sync/atomic" "time" "unsafe" @@ -281,6 +282,7 @@ type dataset struct { mmap mmap.MMap // Memory map itself to unmap before releasing dataset []uint32 // The actual cache data content once sync.Once // Ensures the cache is generated only once + done uint32 // Atomic flag to determine generation status } // newDataset creates a new ethash mining dataset and returns it as a plain Go @@ -292,6 +294,9 @@ func newDataset(epoch uint64) interface{} { // generate ensures that the dataset content is generated before use. func (d *dataset) generate(dir string, limit int, test bool) { d.once.Do(func() { + // Mark the dataset generated after we're done. This is needed for remote + defer atomic.StoreUint32(&d.done, 1) + csize := cacheSize(d.epoch*epochLength + 1) dsize := datasetSize(d.epoch*epochLength + 1) seed := seedHash(d.epoch*epochLength + 1) @@ -306,6 +311,8 @@ func (d *dataset) generate(dir string, limit int, test bool) { d.dataset = make([]uint32, dsize/4) generateDataset(d.dataset, d.epoch, cache) + + return } // Disk storage is needed, this will get fancy var endian string @@ -348,6 +355,13 @@ func (d *dataset) generate(dir string, limit int, test bool) { }) } +// generated returns whether this particular dataset finished generating already +// or not (it may not have been started at all). This is useful for remote miners +// to default to verification caches instead of blocking on DAG generations. +func (d *dataset) generated() bool { + return atomic.LoadUint32(&d.done) == 1 +} + // finalizer closes any file handlers and memory maps open. func (d *dataset) finalizer() { if d.mmap != nil { @@ -589,20 +603,34 @@ func (ethash *Ethash) cache(block uint64) *cache { // dataset tries to retrieve a mining dataset for the specified block number // by first checking against a list of in-memory datasets, then against DAGs // stored on disk, and finally generating one if none can be found. -func (ethash *Ethash) dataset(block uint64) *dataset { +// +// If async is specified, not only the future but the current DAG is also +// generates on a background thread. +func (ethash *Ethash) dataset(block uint64, async bool) *dataset { + // Retrieve the requested ethash dataset epoch := block / epochLength currentI, futureI := ethash.datasets.get(epoch) current := currentI.(*dataset) - // Wait for generation finish. - current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) + // If async is specified, generate everything in a background thread + if async && !current.generated() { + go func() { + current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) - // If we need a new future dataset, now's a good time to regenerate it. - if futureI != nil { - future := futureI.(*dataset) - go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) + if futureI != nil { + future := futureI.(*dataset) + future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) + } + }() + } else { + // Either blocking generation was requested, or already done + current.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) + + if futureI != nil { + future := futureI.(*dataset) + go future.generate(ethash.config.DatasetDir, ethash.config.DatasetsOnDisk, ethash.config.PowMode == ModeTest) + } } - return current } diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index 03d8484739..c3b2c86d12 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -114,7 +114,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s hash = header.HashNoNonce().Bytes() target = new(big.Int).Div(two256, header.Difficulty) number = header.Number.Uint64() - dataset = ethash.dataset(number) + dataset = ethash.dataset(number, false) ) // Start generating random nonces until we abort or find a good one var ( @@ -233,21 +233,22 @@ func (ethash *Ethash) remote(notify []string) { log.Info("Work submitted but none pending", "hash", hash) return false } - // Verify the correctness of submitted result. header := block.Header() header.Nonce = nonce header.MixDigest = mixDigest - if err := ethash.VerifySeal(nil, header); err != nil { - log.Warn("Invalid proof-of-work submitted", "hash", hash, "err", err) + + start := time.Now() + if err := ethash.verifySeal(nil, header, true); err != nil { + log.Warn("Invalid proof-of-work submitted", "hash", hash, "elapsed", time.Since(start), "err", err) return false } - // Make sure the result channel is created. if ethash.resultCh == nil { log.Warn("Ethash result channel is empty, submitted mining result is rejected") return false } + log.Trace("Verified correct proof-of-work", "hash", hash, "elapsed", time.Since(start)) // Solutions seems to be valid, return to the miner and notify acceptance. select { From e8752f4e9f9be3d2932cd4835a5d72d17ac2338b Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 15 Aug 2018 17:41:52 +0200 Subject: [PATCH 13/96] cmd/swarm, swarm: added access control functionality (#17404) Co-authored-by: Janos Guljas Co-authored-by: Anton Evangelatov Co-authored-by: Balint Gabor --- cmd/swarm/access.go | 219 ++++++++++++ cmd/swarm/access_test.go | 581 ++++++++++++++++++++++++++++++++ cmd/swarm/config.go | 1 + cmd/swarm/download.go | 40 ++- cmd/swarm/list.go | 2 +- cmd/swarm/main.go | 95 +++++- cmd/swarm/run_test.go | 25 +- swarm/api/act.go | 468 +++++++++++++++++++++++++ swarm/api/api.go | 135 +++++--- swarm/api/api_test.go | 68 +++- swarm/api/client/client.go | 48 ++- swarm/api/client/client_test.go | 4 +- swarm/api/encrypt.go | 76 +++++ swarm/api/filesystem.go | 4 +- swarm/api/filesystem_test.go | 4 +- swarm/api/http/middleware.go | 12 +- swarm/api/http/response.go | 2 +- swarm/api/http/server.go | 111 +++--- swarm/api/manifest.go | 69 ++-- swarm/api/manifest_test.go | 8 +- swarm/api/storage.go | 6 +- swarm/api/uri.go | 13 + swarm/fuse/swarmfs_test.go | 2 +- swarm/network_test.go | 2 +- swarm/sctx/sctx.go | 15 + swarm/swarm.go | 4 +- swarm/testutil/http.go | 2 +- 27 files changed, 1829 insertions(+), 187 deletions(-) create mode 100644 cmd/swarm/access.go create mode 100644 cmd/swarm/access_test.go create mode 100644 swarm/api/act.go create mode 100644 swarm/api/encrypt.go diff --git a/cmd/swarm/access.go b/cmd/swarm/access.go new file mode 100644 index 0000000000..12cfbfc1a4 --- /dev/null +++ b/cmd/swarm/access.go @@ -0,0 +1,219 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . +package main + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "strings" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/api/client" + "gopkg.in/urfave/cli.v1" +) + +var salt = make([]byte, 32) + +func init() { + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + panic("reading from crypto/rand failed: " + err.Error()) + } +} + +func accessNewPass(ctx *cli.Context) { + args := ctx.Args() + if len(args) != 1 { + utils.Fatalf("Expected 1 argument - the ref") + } + + var ( + ae *api.AccessEntry + accessKey []byte + err error + ref = args[0] + password = getPassPhrase("", 0, makePasswordList(ctx)) + dryRun = ctx.Bool(SwarmDryRunFlag.Name) + ) + accessKey, ae, err = api.DoPasswordNew(ctx, password, salt) + if err != nil { + utils.Fatalf("error getting session key: %v", err) + } + m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae) + if dryRun { + err = printManifests(m, nil) + if err != nil { + utils.Fatalf("had an error printing the manifests: %v", err) + } + } else { + utils.Fatalf("uploading manifests") + err = uploadManifests(ctx, m, nil) + if err != nil { + utils.Fatalf("had an error uploading the manifests: %v", err) + } + } +} + +func accessNewPK(ctx *cli.Context) { + args := ctx.Args() + if len(args) != 1 { + utils.Fatalf("Expected 1 argument - the ref") + } + + var ( + ae *api.AccessEntry + sessionKey []byte + err error + ref = args[0] + privateKey = getPrivKey(ctx) + granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name) + dryRun = ctx.Bool(SwarmDryRunFlag.Name) + ) + sessionKey, ae, err = api.DoPKNew(ctx, privateKey, granteePublicKey, salt) + if err != nil { + utils.Fatalf("error getting session key: %v", err) + } + m, err := api.GenerateAccessControlManifest(ctx, ref, sessionKey, ae) + if dryRun { + err = printManifests(m, nil) + if err != nil { + utils.Fatalf("had an error printing the manifests: %v", err) + } + } else { + err = uploadManifests(ctx, m, nil) + if err != nil { + utils.Fatalf("had an error uploading the manifests: %v", err) + } + } +} + +func accessNewACT(ctx *cli.Context) { + args := ctx.Args() + if len(args) != 1 { + utils.Fatalf("Expected 1 argument - the ref") + } + + var ( + ae *api.AccessEntry + actManifest *api.Manifest + accessKey []byte + err error + ref = args[0] + grantees = []string{} + actFilename = ctx.String(SwarmAccessGrantKeysFlag.Name) + privateKey = getPrivKey(ctx) + dryRun = ctx.Bool(SwarmDryRunFlag.Name) + ) + + bytes, err := ioutil.ReadFile(actFilename) + if err != nil { + utils.Fatalf("had an error reading the grantee public key list") + } + grantees = strings.Split(string(bytes), "\n") + accessKey, ae, actManifest, err = api.DoACTNew(ctx, privateKey, salt, grantees) + if err != nil { + utils.Fatalf("error generating ACT manifest: %v", err) + } + + if err != nil { + utils.Fatalf("error getting session key: %v", err) + } + m, err := api.GenerateAccessControlManifest(ctx, ref, accessKey, ae) + if err != nil { + utils.Fatalf("error generating root access manifest: %v", err) + } + + if dryRun { + err = printManifests(m, actManifest) + if err != nil { + utils.Fatalf("had an error printing the manifests: %v", err) + } + } else { + err = uploadManifests(ctx, m, actManifest) + if err != nil { + utils.Fatalf("had an error uploading the manifests: %v", err) + } + } +} + +func printManifests(rootAccessManifest, actManifest *api.Manifest) error { + js, err := json.Marshal(rootAccessManifest) + if err != nil { + return err + } + fmt.Println(string(js)) + + if actManifest != nil { + js, err := json.Marshal(actManifest) + if err != nil { + return err + } + fmt.Println(string(js)) + } + return nil +} + +func uploadManifests(ctx *cli.Context, rootAccessManifest, actManifest *api.Manifest) error { + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := client.NewClient(bzzapi) + + var ( + key string + err error + ) + if actManifest != nil { + key, err = client.UploadManifest(actManifest, false) + if err != nil { + return err + } + + rootAccessManifest.Entries[0].Access.Act = key + } + key, err = client.UploadManifest(rootAccessManifest, false) + if err != nil { + return err + } + fmt.Println(key) + return nil +} + +// makePasswordList reads password lines from the file specified by the global --password flag +// and also by the same subcommand --password flag. +// This function ia a fork of utils.MakePasswordList to lookup cli context for subcommand. +// Function ctx.SetGlobal is not setting the global flag value that can be accessed +// by ctx.GlobalString using the current version of cli package. +func makePasswordList(ctx *cli.Context) []string { + path := ctx.GlobalString(utils.PasswordFileFlag.Name) + if path == "" { + path = ctx.String(utils.PasswordFileFlag.Name) + if path == "" { + return nil + } + } + text, err := ioutil.ReadFile(path) + if err != nil { + utils.Fatalf("Failed to read password file: %v", err) + } + lines := strings.Split(string(text), "\n") + // Sanitise DOS line endings. + for i := range lines { + lines[i] = strings.TrimRight(lines[i], "\r") + } + return lines +} diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go new file mode 100644 index 0000000000..163eb2b4d6 --- /dev/null +++ b/cmd/swarm/access_test.go @@ -0,0 +1,581 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . +package main + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "encoding/json" + "io" + "io/ioutil" + gorand "math/rand" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/api" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" +) + +// TestAccessPassword tests for the correct creation of an ACT manifest protected by a password. +// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry +// The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded +// is then fetched through 2nd node. since the tested code is not key-aware - we can just +// fetch from the 2nd node using HTTP BasicAuth +func TestAccessPassword(t *testing.T) { + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + proxyNode := cluster.Nodes[0] + + // create a tmp file + tmp, err := ioutil.TempDir("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + // write data to file + data := "notsorandomdata" + dataFilename := filepath.Join(tmp, "data.txt") + + err = ioutil.WriteFile(dataFilename, []byte(data), 0666) + if err != nil { + t.Fatal(err) + } + + hashRegexp := `[a-f\d]{128}` + + // upload the file with 'swarm up' and expect a hash + up := runSwarm(t, + "--bzzapi", + proxyNode.URL, //it doesn't matter through which node we upload content + "up", + "--encrypt", + dataFilename) + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + + if len(matches) < 1 { + t.Fatal("no matches found") + } + + ref := matches[0] + + password := "smth" + passwordFilename := filepath.Join(tmp, "password.txt") + + err = ioutil.WriteFile(passwordFilename, []byte(password), 0666) + if err != nil { + t.Fatal(err) + } + + up = runSwarm(t, + "access", + "new", + "pass", + "--dry-run", + "--password", + passwordFilename, + ref, + ) + + _, matches = up.ExpectRegexp(".+") + up.ExpectExit() + + if len(matches) == 0 { + t.Fatalf("stdout not matched") + } + + var m api.Manifest + + err = json.Unmarshal([]byte(matches[0]), &m) + if err != nil { + t.Fatalf("unmarshal manifest: %v", err) + } + + if len(m.Entries) != 1 { + t.Fatalf("expected one manifest entry, got %v", len(m.Entries)) + } + + e := m.Entries[0] + + ct := "application/bzz-manifest+json" + if e.ContentType != ct { + t.Errorf("expected %q content type, got %q", ct, e.ContentType) + } + + if e.Access == nil { + t.Fatal("manifest access is nil") + } + + a := e.Access + + if a.Type != "pass" { + t.Errorf(`got access type %q, expected "pass"`, a.Type) + } + if len(a.Salt) < 32 { + t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt)) + } + if a.KdfParams == nil { + t.Fatal("manifest access kdf params is nil") + } + + client := swarm.NewClient(cluster.Nodes[0].URL) + + hash, err := client.UploadManifest(&m, false) + if err != nil { + t.Fatal(err) + } + + httpClient := &http.Client{} + + url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash + response, err := httpClient.Get(url) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusUnauthorized { + t.Fatal("should be a 401") + } + authHeader := response.Header.Get("WWW-Authenticate") + if authHeader == "" { + t.Fatal("should be something here") + } + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.SetBasicAuth("", password) + + response, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + t.Errorf("expected status %v, got %v", http.StatusOK, response.StatusCode) + } + d, err := ioutil.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if string(d) != data { + t.Errorf("expected decrypted data %q, got %q", data, string(d)) + } + + wrongPasswordFilename := filepath.Join(tmp, "password-wrong.txt") + + err = ioutil.WriteFile(wrongPasswordFilename, []byte("just wr0ng"), 0666) + if err != nil { + t.Fatal(err) + } + + //download file with 'swarm down' with wrong password + up = runSwarm(t, + "--bzzapi", + proxyNode.URL, + "down", + "bzz:/"+hash, + tmp, + "--password", + wrongPasswordFilename) + + _, matches = up.ExpectRegexp("unauthorized") + if len(matches) != 1 && matches[0] != "unauthorized" { + t.Fatal(`"unauthorized" not found in output"`) + } + up.ExpectExit() +} + +// TestAccessPK tests for the correct creation of an ACT manifest between two parties (publisher and grantee). +// The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry +// The parties participating - node (publisher), uploads to second node (which is also the grantee) then disappears. +// Content which was uploaded is then fetched through the grantee's http proxy. Since the tested code is private-key aware, +// the test will fail if the proxy's given private key is not granted on the ACT. +func TestAccessPK(t *testing.T) { + // Setup Swarm and upload a test file to it + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + // create a tmp file + tmp, err := ioutil.TempFile("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer tmp.Close() + defer os.Remove(tmp.Name()) + + // write data to file + data := "notsorandomdata" + _, err = io.WriteString(tmp, data) + if err != nil { + t.Fatal(err) + } + + hashRegexp := `[a-f\d]{128}` + + // upload the file with 'swarm up' and expect a hash + up := runSwarm(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + "--encrypt", + tmp.Name()) + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + + if len(matches) < 1 { + t.Fatal("no matches found") + } + + ref := matches[0] + + pk := cluster.Nodes[0].PrivateKey + granteePubKey := crypto.CompressPubkey(&pk.PublicKey) + + publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp") + if err != nil { + t.Fatal(err) + } + + passFile, err := ioutil.TempFile("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer passFile.Close() + defer os.Remove(passFile.Name()) + _, err = io.WriteString(passFile, testPassphrase) + if err != nil { + t.Fatal(err) + } + _, publisherAccount := getTestAccount(t, publisherDir) + up = runSwarm(t, + "--bzzaccount", + publisherAccount.Address.String(), + "--password", + passFile.Name(), + "--datadir", + publisherDir, + "--bzzapi", + cluster.Nodes[0].URL, + "access", + "new", + "pk", + "--dry-run", + "--grant-key", + hex.EncodeToString(granteePubKey), + ref, + ) + + _, matches = up.ExpectRegexp(".+") + up.ExpectExit() + + if len(matches) == 0 { + t.Fatalf("stdout not matched") + } + + var m api.Manifest + + err = json.Unmarshal([]byte(matches[0]), &m) + if err != nil { + t.Fatalf("unmarshal manifest: %v", err) + } + + if len(m.Entries) != 1 { + t.Fatalf("expected one manifest entry, got %v", len(m.Entries)) + } + + e := m.Entries[0] + + ct := "application/bzz-manifest+json" + if e.ContentType != ct { + t.Errorf("expected %q content type, got %q", ct, e.ContentType) + } + + if e.Access == nil { + t.Fatal("manifest access is nil") + } + + a := e.Access + + if a.Type != "pk" { + t.Errorf(`got access type %q, expected "pk"`, a.Type) + } + if len(a.Salt) < 32 { + t.Errorf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt)) + } + if a.KdfParams != nil { + t.Fatal("manifest access kdf params should be nil") + } + + client := swarm.NewClient(cluster.Nodes[0].URL) + + hash, err := client.UploadManifest(&m, false) + if err != nil { + t.Fatal(err) + } + + httpClient := &http.Client{} + + url := cluster.Nodes[0].URL + "/" + "bzz:/" + hash + response, err := httpClient.Get(url) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusOK { + t.Fatal("should be a 200") + } + d, err := ioutil.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if string(d) != data { + t.Errorf("expected decrypted data %q, got %q", data, string(d)) + } +} + +// TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control +// the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data +// set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through +// one of the nodes then disappears. +func TestAccessACT(t *testing.T) { + // Setup Swarm and upload a test file to it + cluster := newTestCluster(t, 3) + defer cluster.Shutdown() + + var uploadThroughNode = cluster.Nodes[0] + client := swarm.NewClient(uploadThroughNode.URL) + + r1 := gorand.New(gorand.NewSource(time.Now().UnixNano())) + nodeToSkip := r1.Intn(3) // a number between 0 and 2 (node indices in `cluster`) + // create a tmp file + tmp, err := ioutil.TempFile("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer tmp.Close() + defer os.Remove(tmp.Name()) + + // write data to file + data := "notsorandomdata" + _, err = io.WriteString(tmp, data) + if err != nil { + t.Fatal(err) + } + + hashRegexp := `[a-f\d]{128}` + + // upload the file with 'swarm up' and expect a hash + up := runSwarm(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + "--encrypt", + tmp.Name()) + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + + if len(matches) < 1 { + t.Fatal("no matches found") + } + + ref := matches[0] + grantees := []string{} + for i, v := range cluster.Nodes { + if i == nodeToSkip { + continue + } + pk := v.PrivateKey + granteePubKey := crypto.CompressPubkey(&pk.PublicKey) + grantees = append(grantees, hex.EncodeToString(granteePubKey)) + } + + granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list.csv") + if err != nil { + t.Fatal(err) + } + + _, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n")) + if err != nil { + t.Fatal(err) + } + + defer granteesPubkeyListFile.Close() + defer os.Remove(granteesPubkeyListFile.Name()) + + publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp") + if err != nil { + t.Fatal(err) + } + + passFile, err := ioutil.TempFile("", "swarm-test") + if err != nil { + t.Fatal(err) + } + defer passFile.Close() + defer os.Remove(passFile.Name()) + _, err = io.WriteString(passFile, testPassphrase) + if err != nil { + t.Fatal(err) + } + + _, publisherAccount := getTestAccount(t, publisherDir) + up = runSwarm(t, + "--bzzaccount", + publisherAccount.Address.String(), + "--password", + passFile.Name(), + "--datadir", + publisherDir, + "--bzzapi", + cluster.Nodes[0].URL, + "access", + "new", + "act", + "--grant-keys", + granteesPubkeyListFile.Name(), + ref, + ) + + _, matches = up.ExpectRegexp(`[a-f\d]{64}`) + up.ExpectExit() + + if len(matches) == 0 { + t.Fatalf("stdout not matched") + } + hash := matches[0] + m, _, err := client.DownloadManifest(hash) + if err != nil { + t.Fatalf("unmarshal manifest: %v", err) + } + + if len(m.Entries) != 1 { + t.Fatalf("expected one manifest entry, got %v", len(m.Entries)) + } + + e := m.Entries[0] + + ct := "application/bzz-manifest+json" + if e.ContentType != ct { + t.Errorf("expected %q content type, got %q", ct, e.ContentType) + } + + if e.Access == nil { + t.Fatal("manifest access is nil") + } + + a := e.Access + + if a.Type != "act" { + t.Fatalf(`got access type %q, expected "act"`, a.Type) + } + if len(a.Salt) < 32 { + t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt)) + } + if a.KdfParams != nil { + t.Fatal("manifest access kdf params should be nil") + } + + httpClient := &http.Client{} + + // all nodes except the skipped node should be able to decrypt the content + for i, node := range cluster.Nodes { + log.Debug("trying to fetch from node", "node index", i) + + url := node.URL + "/" + "bzz:/" + hash + response, err := httpClient.Get(url) + if err != nil { + t.Fatal(err) + } + log.Debug("got response from node", "response code", response.StatusCode) + + if i == nodeToSkip { + log.Debug("reached node to skip", "status code", response.StatusCode) + + if response.StatusCode != http.StatusUnauthorized { + t.Fatalf("should be a 401") + } + + continue + } + + if response.StatusCode != http.StatusOK { + t.Fatal("should be a 200") + } + d, err := ioutil.ReadAll(response.Body) + if err != nil { + t.Fatal(err) + } + if string(d) != data { + t.Errorf("expected decrypted data %q, got %q", data, string(d)) + } + } +} + +// TestKeypairSanity is a sanity test for the crypto scheme for ACT. it asserts the correct shared secret according to +// the specs at https://github.com/ethersphere/swarm-docs/blob/eb857afda906c6e7bb90d37f3f334ccce5eef230/act.md +func TestKeypairSanity(t *testing.T) { + salt := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + t.Fatalf("reading from crypto/rand failed: %v", err.Error()) + } + sharedSecret := "a85586744a1ddd56a7ed9f33fa24f40dd745b3a941be296a0d60e329dbdb896d" + + for i, v := range []struct { + publisherPriv string + granteePub string + }{ + { + publisherPriv: "ec5541555f3bc6376788425e9d1a62f55a82901683fd7062c5eddcc373a73459", + granteePub: "0226f213613e843a413ad35b40f193910d26eb35f00154afcde9ded57479a6224a", + }, + { + publisherPriv: "70c7a73011aa56584a0009ab874794ee7e5652fd0c6911cd02f8b6267dd82d2d", + granteePub: "02e6f8d5e28faaa899744972bb847b6eb805a160494690c9ee7197ae9f619181db", + }, + } { + b, _ := hex.DecodeString(v.granteePub) + granteePub, _ := crypto.DecompressPubkey(b) + publisherPrivate, _ := crypto.HexToECDSA(v.publisherPriv) + + ssKey, err := api.NewSessionKeyPK(publisherPrivate, granteePub, salt) + if err != nil { + t.Fatal(err) + } + + hasher := sha3.NewKeccak256() + hasher.Write(salt) + shared, err := hex.DecodeString(sharedSecret) + if err != nil { + t.Fatal(err) + } + hasher.Write(shared) + sum := hasher.Sum(nil) + + if !bytes.Equal(ssKey, sum) { + t.Fatalf("%d: got a session key mismatch", i) + } + } +} diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index cda8c41c32..1183f8bc81 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -78,6 +78,7 @@ const ( SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" + SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" GETH_ENV_DATADIR = "GETH_DATADIR" ) diff --git a/cmd/swarm/download.go b/cmd/swarm/download.go index c2418f744c..91bc2c93ab 100644 --- a/cmd/swarm/download.go +++ b/cmd/swarm/download.go @@ -68,18 +68,36 @@ func download(ctx *cli.Context) { utils.Fatalf("could not parse uri argument: %v", err) } - // assume behaviour according to --recursive switch - if isRecursive { - if err := client.DownloadDirectory(uri.Addr, uri.Path, dest); err != nil { - utils.Fatalf("encoutered an error while downloading directory: %v", err) - } - } else { - // we are downloading a file - log.Debug(fmt.Sprintf("downloading file/path from a manifest. hash: %s, path:%s", uri.Addr, uri.Path)) + dl := func(credentials string) error { + // assume behaviour according to --recursive switch + if isRecursive { + if err := client.DownloadDirectory(uri.Addr, uri.Path, dest, credentials); err != nil { + if err == swarm.ErrUnauthorized { + return err + } + return fmt.Errorf("directory %s: %v", uri.Path, err) + } + } else { + // we are downloading a file + log.Debug("downloading file/path from a manifest", "uri.Addr", uri.Addr, "uri.Path", uri.Path) - err := client.DownloadFile(uri.Addr, uri.Path, dest) - if err != nil { - utils.Fatalf("could not download %s from given address: %s. error: %v", uri.Path, uri.Addr, err) + err := client.DownloadFile(uri.Addr, uri.Path, dest, credentials) + if err != nil { + if err == swarm.ErrUnauthorized { + return err + } + return fmt.Errorf("file %s from address: %s: %v", uri.Path, uri.Addr, err) + } } + return nil + } + if passwords := makePasswordList(ctx); passwords != nil { + password := getPassPhrase(fmt.Sprintf("Downloading %s is restricted", uri), 0, passwords) + err = dl(password) + } else { + err = dl("") + } + if err != nil { + utils.Fatalf("download: %v", err) } } diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go index 57b5517c6e..01b3f4ab6c 100644 --- a/cmd/swarm/list.go +++ b/cmd/swarm/list.go @@ -44,7 +44,7 @@ func list(ctx *cli.Context) { bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") client := swarm.NewClient(bzzapi) - list, err := client.List(manifest, prefix) + list, err := client.List(manifest, prefix, "") if err != nil { utils.Fatalf("Failed to generate file and directory list: %s", err) } diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index ac09ae9981..76be60cb68 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -155,6 +155,14 @@ var ( Name: "defaultpath", Usage: "path to file served for empty url path (none)", } + SwarmAccessGrantKeyFlag = cli.StringFlag{ + Name: "grant-key", + Usage: "grants a given public key access to an ACT", + } + SwarmAccessGrantKeysFlag = cli.StringFlag{ + Name: "grant-keys", + Usage: "grants a given list of public keys in the following file (separated by line breaks) access to an ACT", + } SwarmUpFromStdinFlag = cli.BoolFlag{ Name: "stdin", Usage: "reads data to be uploaded from stdin", @@ -167,6 +175,15 @@ var ( Name: "encrypt", Usage: "use encrypted upload", } + SwarmAccessPasswordFlag = cli.StringFlag{ + Name: "password", + Usage: "Password", + EnvVar: SWARM_ACCESS_PASSWORD, + } + SwarmDryRunFlag = cli.BoolFlag{ + Name: "dry-run", + Usage: "dry-run", + } CorsStringFlag = cli.StringFlag{ Name: "corsdomain", Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", @@ -252,6 +269,61 @@ func init() { Flags: []cli.Flag{SwarmEncryptedFlag}, Description: "uploads a file or directory to swarm using the HTTP API and prints the root hash", }, + { + CustomHelpTemplate: helpTemplate, + Name: "access", + Usage: "encrypts a reference and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root manifest", + Subcommands: []cli.Command{ + { + CustomHelpTemplate: helpTemplate, + Name: "new", + Usage: "encrypts a reference and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + Subcommands: []cli.Command{ + { + Action: accessNewPass, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + utils.PasswordFileFlag, + SwarmDryRunFlag, + }, + Name: "pass", + Usage: "encrypts a reference with a password and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + { + Action: accessNewPK, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + utils.PasswordFileFlag, + SwarmDryRunFlag, + SwarmAccessGrantKeyFlag, + }, + Name: "pk", + Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + { + Action: accessNewACT, + CustomHelpTemplate: helpTemplate, + Flags: []cli.Flag{ + SwarmAccessGrantKeysFlag, + SwarmDryRunFlag, + }, + Name: "act", + Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", + ArgsUsage: "", + Description: "encrypts a reference and embeds it into a root access manifest and prints the resulting manifest", + }, + }, + }, + }, + }, { CustomHelpTemplate: helpTemplate, Name: "resource", @@ -304,16 +376,13 @@ func init() { Description: "Prints the swarm hash of file or directory", }, { - Action: download, - Name: "down", - Flags: []cli.Flag{SwarmRecursiveFlag}, - Usage: "downloads a swarm manifest or a file inside a manifest", - ArgsUsage: " []", - Description: ` -Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries. -`, + Action: download, + Name: "down", + Flags: []cli.Flag{SwarmRecursiveFlag, SwarmAccessPasswordFlag}, + Usage: "downloads a swarm manifest or a file inside a manifest", + ArgsUsage: " []", + Description: `Downloads a swarm bzz uri to the given dir. When no dir is provided, working directory is assumed. --recursive flag is expected when downloading a manifest with multiple entries.`, }, - { Name: "manifest", CustomHelpTemplate: helpTemplate, @@ -413,16 +482,14 @@ pv(1) tool to get a progress bar: Name: "import", Usage: "import chunks from a tar archive into a local chunk database (use - to read from stdin)", ArgsUsage: " ", - Description: ` -Import chunks from a tar archive into a local chunk database (use - to read from stdin). + Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin). swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar The import may be quite large, consider piping the input through the Unix pv(1) tool to get a progress bar: - pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks - -`, + pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`, }, { Action: dbClean, @@ -535,6 +602,7 @@ func version(ctx *cli.Context) error { func bzzd(ctx *cli.Context) error { //build a valid bzzapi.Config from all available sources: //default config, file config, command line and env vars + bzzconfig, err := buildConfig(ctx) if err != nil { utils.Fatalf("unable to configure swarm: %v", err) @@ -557,6 +625,7 @@ func bzzd(ctx *cli.Context) error { if err != nil { utils.Fatalf("can't create node: %v", err) } + //a few steps need to be done after the config phase is completed, //due to overriding behavior initSwarmNode(bzzconfig, stack, ctx) diff --git a/cmd/swarm/run_test.go b/cmd/swarm/run_test.go index 90d3c98ba6..3e766dc10d 100644 --- a/cmd/swarm/run_test.go +++ b/cmd/swarm/run_test.go @@ -18,10 +18,12 @@ package main import ( "context" + "crypto/ecdsa" "fmt" "io/ioutil" "net" "os" + "path" "path/filepath" "runtime" "sync" @@ -175,14 +177,15 @@ func (c *testCluster) Cleanup() { } type testNode struct { - Name string - Addr string - URL string - Enode string - Dir string - IpcPath string - Client *rpc.Client - Cmd *cmdtest.TestCmd + Name string + Addr string + URL string + Enode string + Dir string + IpcPath string + PrivateKey *ecdsa.PrivateKey + Client *rpc.Client + Cmd *cmdtest.TestCmd } const testPassphrase = "swarm-test-passphrase" @@ -289,7 +292,11 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode { func newTestNode(t *testing.T, dir string) *testNode { conf, account := getTestAccount(t, dir) - node := &testNode{Dir: dir} + ks := keystore.NewKeyStore(path.Join(dir, "keystore"), 1<<18, 1) + + pk := decryptStoreAccount(ks, account.Address.Hex(), []string{testPassphrase}) + + node := &testNode{Dir: dir, PrivateKey: pk} // assign ports ports, err := getAvailableTCPPorts(2) diff --git a/swarm/api/act.go b/swarm/api/act.go new file mode 100644 index 0000000000..b1a5947831 --- /dev/null +++ b/swarm/api/act.go @@ -0,0 +1,468 @@ +package api + +import ( + "context" + "crypto/ecdsa" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/ecies" + "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/sctx" + "github.com/ethereum/go-ethereum/swarm/storage" + "golang.org/x/crypto/scrypt" + cli "gopkg.in/urfave/cli.v1" +) + +var ( + ErrDecrypt = errors.New("cant decrypt - forbidden") + ErrUnknownAccessType = errors.New("unknown access type (or not implemented)") + ErrDecryptDomainForbidden = errors.New("decryption request domain forbidden - can only decrypt on localhost") + AllowedDecryptDomains = []string{ + "localhost", + "127.0.0.1", + } +) + +const EMPTY_CREDENTIALS = "" + +type AccessEntry struct { + Type AccessType + Publisher string + Salt []byte + Act string + KdfParams *KdfParams +} + +type DecryptFunc func(*ManifestEntry) error + +func (a *AccessEntry) MarshalJSON() (out []byte, err error) { + + return json.Marshal(struct { + Type AccessType `json:"type,omitempty"` + Publisher string `json:"publisher,omitempty"` + Salt string `json:"salt,omitempty"` + Act string `json:"act,omitempty"` + KdfParams *KdfParams `json:"kdf_params,omitempty"` + }{ + Type: a.Type, + Publisher: a.Publisher, + Salt: hex.EncodeToString(a.Salt), + Act: a.Act, + KdfParams: a.KdfParams, + }) + +} + +func (a *AccessEntry) UnmarshalJSON(value []byte) error { + v := struct { + Type AccessType `json:"type,omitempty"` + Publisher string `json:"publisher,omitempty"` + Salt string `json:"salt,omitempty"` + Act string `json:"act,omitempty"` + KdfParams *KdfParams `json:"kdf_params,omitempty"` + }{} + + err := json.Unmarshal(value, &v) + if err != nil { + return err + } + a.Act = v.Act + a.KdfParams = v.KdfParams + a.Publisher = v.Publisher + a.Salt, err = hex.DecodeString(v.Salt) + if err != nil { + return err + } + if len(a.Salt) != 32 { + return errors.New("salt should be 32 bytes long") + } + a.Type = v.Type + return nil +} + +type KdfParams struct { + N int `json:"n"` + P int `json:"p"` + R int `json:"r"` +} + +type AccessType string + +const AccessTypePass = AccessType("pass") +const AccessTypePK = AccessType("pk") +const AccessTypeACT = AccessType("act") + +func NewAccessEntryPassword(salt []byte, kdfParams *KdfParams) (*AccessEntry, error) { + if len(salt) != 32 { + return nil, fmt.Errorf("salt should be 32 bytes long") + } + return &AccessEntry{ + Type: AccessTypePass, + Salt: salt, + KdfParams: kdfParams, + }, nil +} + +func NewAccessEntryPK(publisher string, salt []byte) (*AccessEntry, error) { + if len(publisher) != 66 { + return nil, fmt.Errorf("publisher should be 66 characters long, got %d", len(publisher)) + } + if len(salt) != 32 { + return nil, fmt.Errorf("salt should be 32 bytes long") + } + return &AccessEntry{ + Type: AccessTypePK, + Publisher: publisher, + Salt: salt, + }, nil +} + +func NewAccessEntryACT(publisher string, salt []byte, act string) (*AccessEntry, error) { + if len(salt) != 32 { + return nil, fmt.Errorf("salt should be 32 bytes long") + } + if len(publisher) != 66 { + return nil, fmt.Errorf("publisher should be 66 characters long") + } + + return &AccessEntry{ + Type: AccessTypeACT, + Publisher: publisher, + Salt: salt, + Act: act, + }, nil +} + +func NOOPDecrypt(*ManifestEntry) error { + return nil +} + +var DefaultKdfParams = NewKdfParams(262144, 1, 8) + +func NewKdfParams(n, p, r int) *KdfParams { + + return &KdfParams{ + N: n, + P: p, + R: r, + } +} + +// NewSessionKeyPassword creates a session key based on a shared secret (password) and the given salt +// and kdf parameters in the access entry +func NewSessionKeyPassword(password string, accessEntry *AccessEntry) ([]byte, error) { + if accessEntry.Type != AccessTypePass { + return nil, errors.New("incorrect access entry type") + } + return scrypt.Key( + []byte(password), + accessEntry.Salt, + accessEntry.KdfParams.N, + accessEntry.KdfParams.R, + accessEntry.KdfParams.P, + 32, + ) +} + +// NewSessionKeyPK creates a new ACT Session Key using an ECDH shared secret for the given key pair and the given salt value +func NewSessionKeyPK(private *ecdsa.PrivateKey, public *ecdsa.PublicKey, salt []byte) ([]byte, error) { + granteePubEcies := ecies.ImportECDSAPublic(public) + privateKey := ecies.ImportECDSA(private) + + bytes, err := privateKey.GenerateShared(granteePubEcies, 16, 16) + if err != nil { + return nil, err + } + bytes = append(salt, bytes...) + sessionKey := crypto.Keccak256(bytes) + return sessionKey, nil +} + +func (a *API) NodeSessionKey(privateKey *ecdsa.PrivateKey, publicKey *ecdsa.PublicKey, salt []byte) ([]byte, error) { + return NewSessionKeyPK(privateKey, publicKey, salt) +} +func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.PrivateKey) DecryptFunc { + return func(m *ManifestEntry) error { + if m.Access == nil { + return nil + } + + allowed := false + requestDomain := sctx.GetHost(ctx) + for _, v := range AllowedDecryptDomains { + if strings.Contains(requestDomain, v) { + allowed = true + } + } + + if !allowed { + return ErrDecryptDomainForbidden + } + + switch m.Access.Type { + case "pass": + if credentials != "" { + key, err := NewSessionKeyPassword(credentials, m.Access) + if err != nil { + return err + } + + ref, err := hex.DecodeString(m.Hash) + if err != nil { + return err + } + + enc := NewRefEncryption(len(ref) - 8) + decodedRef, err := enc.Decrypt(ref, key) + if err != nil { + return ErrDecrypt + } + + m.Hash = hex.EncodeToString(decodedRef) + m.Access = nil + return nil + } + return ErrDecrypt + case "pk": + publisherBytes, err := hex.DecodeString(m.Access.Publisher) + if err != nil { + return ErrDecrypt + } + publisher, err := crypto.DecompressPubkey(publisherBytes) + if err != nil { + return ErrDecrypt + } + key, err := a.NodeSessionKey(pk, publisher, m.Access.Salt) + if err != nil { + return ErrDecrypt + } + ref, err := hex.DecodeString(m.Hash) + if err != nil { + return err + } + + enc := NewRefEncryption(len(ref) - 8) + decodedRef, err := enc.Decrypt(ref, key) + if err != nil { + return ErrDecrypt + } + + m.Hash = hex.EncodeToString(decodedRef) + m.Access = nil + return nil + case "act": + publisherBytes, err := hex.DecodeString(m.Access.Publisher) + if err != nil { + return ErrDecrypt + } + publisher, err := crypto.DecompressPubkey(publisherBytes) + if err != nil { + return ErrDecrypt + } + + sessionKey, err := a.NodeSessionKey(pk, publisher, m.Access.Salt) + if err != nil { + return ErrDecrypt + } + + hasher := sha3.NewKeccak256() + hasher.Write(append(sessionKey, 0)) + lookupKey := hasher.Sum(nil) + + hasher.Reset() + + hasher.Write(append(sessionKey, 1)) + accessKeyDecryptionKey := hasher.Sum(nil) + + lk := hex.EncodeToString(lookupKey) + list, err := a.GetManifestList(ctx, NOOPDecrypt, storage.Address(common.Hex2Bytes(m.Access.Act)), lk) + + found := "" + for _, v := range list.Entries { + if v.Path == lk { + found = v.Hash + } + } + + if found == "" { + return ErrDecrypt + } + + v, err := hex.DecodeString(found) + if err != nil { + return err + } + enc := NewRefEncryption(len(v) - 8) + decodedRef, err := enc.Decrypt(v, accessKeyDecryptionKey) + if err != nil { + return ErrDecrypt + } + + ref, err := hex.DecodeString(m.Hash) + if err != nil { + return err + } + + enc = NewRefEncryption(len(ref) - 8) + decodedMainRef, err := enc.Decrypt(ref, decodedRef) + if err != nil { + return ErrDecrypt + } + m.Hash = hex.EncodeToString(decodedMainRef) + m.Access = nil + return nil + } + return ErrUnknownAccessType + } +} + +func GenerateAccessControlManifest(ctx *cli.Context, ref string, accessKey []byte, ae *AccessEntry) (*Manifest, error) { + refBytes, err := hex.DecodeString(ref) + if err != nil { + return nil, err + } + // encrypt ref with accessKey + enc := NewRefEncryption(len(refBytes)) + encrypted, err := enc.Encrypt(refBytes, accessKey) + if err != nil { + return nil, err + } + + m := &Manifest{ + Entries: []ManifestEntry{ + { + Hash: hex.EncodeToString(encrypted), + ContentType: ManifestType, + ModTime: time.Now(), + Access: ae, + }, + }, + } + + return m, nil +} + +func DoPKNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { + if granteePublicKey == "" { + return nil, nil, errors.New("need a grantee Public Key") + } + b, err := hex.DecodeString(granteePublicKey) + if err != nil { + log.Error("error decoding grantee public key", "err", err) + return nil, nil, err + } + + granteePub, err := crypto.DecompressPubkey(b) + if err != nil { + log.Error("error decompressing grantee public key", "err", err) + return nil, nil, err + } + + sessionKey, err = NewSessionKeyPK(privateKey, granteePub, salt) + if err != nil { + log.Error("error getting session key", "err", err) + return nil, nil, err + } + + ae, err = NewAccessEntryPK(hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)), salt) + if err != nil { + log.Error("error generating access entry", "err", err) + return nil, nil, err + } + + return sessionKey, ae, nil +} + +func DoACTNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees []string) (accessKey []byte, ae *AccessEntry, actManifest *Manifest, err error) { + if len(grantees) == 0 { + return nil, nil, nil, errors.New("did not get any grantee public keys") + } + + publisherPub := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)) + grantees = append(grantees, publisherPub) + + accessKey = make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, salt); err != nil { + panic("reading from crypto/rand failed: " + err.Error()) + } + if _, err := io.ReadFull(rand.Reader, accessKey); err != nil { + panic("reading from crypto/rand failed: " + err.Error()) + } + + lookupPathEncryptedAccessKeyMap := make(map[string]string) + i := 0 + for _, v := range grantees { + i++ + if v == "" { + return nil, nil, nil, errors.New("need a grantee Public Key") + } + b, err := hex.DecodeString(v) + if err != nil { + log.Error("error decoding grantee public key", "err", err) + return nil, nil, nil, err + } + + granteePub, err := crypto.DecompressPubkey(b) + if err != nil { + log.Error("error decompressing grantee public key", "err", err) + return nil, nil, nil, err + } + sessionKey, err := NewSessionKeyPK(privateKey, granteePub, salt) + + hasher := sha3.NewKeccak256() + hasher.Write(append(sessionKey, 0)) + lookupKey := hasher.Sum(nil) + + hasher.Reset() + hasher.Write(append(sessionKey, 1)) + + accessKeyEncryptionKey := hasher.Sum(nil) + + enc := NewRefEncryption(len(accessKey)) + encryptedAccessKey, err := enc.Encrypt(accessKey, accessKeyEncryptionKey) + + lookupPathEncryptedAccessKeyMap[hex.EncodeToString(lookupKey)] = hex.EncodeToString(encryptedAccessKey) + } + + m := &Manifest{ + Entries: []ManifestEntry{}, + } + + for k, v := range lookupPathEncryptedAccessKeyMap { + m.Entries = append(m.Entries, ManifestEntry{ + Path: k, + Hash: v, + ContentType: "text/plain", + }) + } + + ae, err = NewAccessEntryACT(hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)), salt, "") + if err != nil { + return nil, nil, nil, err + } + + return accessKey, ae, m, nil +} + +func DoPasswordNew(ctx *cli.Context, password string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { + ae, err = NewAccessEntryPassword(salt, DefaultKdfParams) + if err != nil { + return nil, nil, err + } + + sessionKey, err = NewSessionKeyPassword(password, ae) + if err != nil { + return nil, nil, err + } + return sessionKey, ae, nil +} diff --git a/swarm/api/api.go b/swarm/api/api.go index 99d971b105..adf469cfaa 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -19,6 +19,9 @@ package api import ( "archive/tar" "context" + "crypto/ecdsa" + "encoding/hex" + "errors" "fmt" "io" "math/big" @@ -43,6 +46,10 @@ import ( opentracing "github.com/opentracing/opentracing-go" ) +var ( + ErrNotFound = errors.New("not found") +) + var ( apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil) apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil) @@ -227,14 +234,18 @@ type API struct { resource *mru.Handler fileStore *storage.FileStore dns Resolver + Decryptor func(context.Context, string) DecryptFunc } // NewAPI the api constructor initialises a new API instance. -func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Handler) (self *API) { +func NewAPI(fileStore *storage.FileStore, dns Resolver, resourceHandler *mru.Handler, pk *ecdsa.PrivateKey) (self *API) { self = &API{ fileStore: fileStore, dns: dns, resource: resourceHandler, + Decryptor: func(ctx context.Context, credentials string) DecryptFunc { + return self.doDecrypt(ctx, credentials, pk) + }, } return } @@ -260,8 +271,30 @@ func (a *API) Store(ctx context.Context, data io.Reader, size int64, toEncrypt b // ErrResolve is returned when an URI cannot be resolved from ENS. type ErrResolve error +// Resolve a name into a content-addressed hash +// where address could be an ENS name, or a content addressed hash +func (a *API) Resolve(ctx context.Context, address string) (storage.Address, error) { + // if DNS is not configured, return an error + if a.dns == nil { + if hashMatcher.MatchString(address) { + return common.Hex2Bytes(address), nil + } + apiResolveFail.Inc(1) + return nil, fmt.Errorf("no DNS to resolve name: %q", address) + } + // try and resolve the address + resolved, err := a.dns.Resolve(address) + if err != nil { + if hashMatcher.MatchString(address) { + return common.Hex2Bytes(address), nil + } + return nil, err + } + return resolved[:], nil +} + // Resolve resolves a URI to an Address using the MultiResolver. -func (a *API) Resolve(ctx context.Context, uri *URI) (storage.Address, error) { +func (a *API) ResolveURI(ctx context.Context, uri *URI, credentials string) (storage.Address, error) { apiResolveCount.Inc(1) log.Trace("resolving", "uri", uri.Addr) @@ -280,28 +313,44 @@ func (a *API) Resolve(ctx context.Context, uri *URI) (storage.Address, error) { return key, nil } - // if DNS is not configured, check if the address is a hash - if a.dns == nil { - key := uri.Address() - if key == nil { - apiResolveFail.Inc(1) - return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr) - } - return key, nil - } - - // try and resolve the address - resolved, err := a.dns.Resolve(uri.Addr) - if err == nil { - return resolved[:], nil - } - - key := uri.Address() - if key == nil { - apiResolveFail.Inc(1) + addr, err := a.Resolve(ctx, uri.Addr) + if err != nil { return nil, err } - return key, nil + + if uri.Path == "" { + return addr, nil + } + walker, err := a.NewManifestWalker(ctx, addr, a.Decryptor(ctx, credentials), nil) + if err != nil { + return nil, err + } + var entry *ManifestEntry + walker.Walk(func(e *ManifestEntry) error { + // if the entry matches the path, set entry and stop + // the walk + if e.Path == uri.Path { + entry = e + // return an error to cancel the walk + return errors.New("found") + } + // ignore non-manifest files + if e.ContentType != ManifestType { + return nil + } + // if the manifest's path is a prefix of the + // requested path, recurse into it by returning + // nil and continuing the walk + if strings.HasPrefix(uri.Path, e.Path) { + return nil + } + return ErrSkipManifest + }) + if entry == nil { + return nil, errors.New("not found") + } + addr = storage.Address(common.Hex2Bytes(entry.Hash)) + return addr, nil } // Put provides singleton manifest creation on top of FileStore store @@ -332,10 +381,10 @@ func (a *API) Put(ctx context.Context, content string, contentType string, toEnc // Get uses iterative manifest retrieval and prefix matching // to resolve basePath to content using FileStore retrieve // it returns a section reader, mimeType, status, the key of the actual content and an error -func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) { +func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage.Address, path string) (reader storage.LazySectionReader, mimeType string, status int, contentAddr storage.Address, err error) { log.Debug("api.get", "key", manifestAddr, "path", path) apiGetCount.Inc(1) - trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil) + trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound @@ -347,6 +396,16 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string if entry != nil { log.Debug("trie got entry", "key", manifestAddr, "path", path, "entry.Hash", entry.Hash) + + if entry.ContentType == ManifestType { + log.Debug("entry is manifest", "key", manifestAddr, "new key", entry.Hash) + adr, err := hex.DecodeString(entry.Hash) + if err != nil { + return nil, "", 0, nil, err + } + return a.Get(ctx, decrypt, adr, entry.Path) + } + // we need to do some extra work if this is a mutable resource manifest if entry.ContentType == ResourceContentType { @@ -398,7 +457,7 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string log.Trace("resource is multihash", "key", manifestAddr) // get the manifest the multihash digest points to - trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil) + trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, decrypt) if err != nil { apiGetNotFound.Inc(1) status = http.StatusNotFound @@ -451,7 +510,7 @@ func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Add apiDeleteFail.Inc(1) return nil, err } - key, err := a.Resolve(ctx, uri) + key, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) if err != nil { return nil, err @@ -470,13 +529,13 @@ func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Add // GetDirectoryTar fetches a requested directory as a tarstream // it returns an io.Reader and an error. Do not forget to Close() the returned ReadCloser -func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, error) { +func (a *API) GetDirectoryTar(ctx context.Context, decrypt DecryptFunc, uri *URI) (io.ReadCloser, error) { apiGetTarCount.Inc(1) - addr, err := a.Resolve(ctx, uri) + addr, err := a.Resolve(ctx, uri.Addr) if err != nil { return nil, err } - walker, err := a.NewManifestWalker(ctx, addr, nil) + walker, err := a.NewManifestWalker(ctx, addr, decrypt, nil) if err != nil { apiGetTarFail.Inc(1) return nil, err @@ -542,9 +601,9 @@ func (a *API) GetDirectoryTar(ctx context.Context, uri *URI) (io.ReadCloser, err // GetManifestList lists the manifest entries for the specified address and prefix // and returns it as a ManifestList -func (a *API) GetManifestList(ctx context.Context, addr storage.Address, prefix string) (list ManifestList, err error) { +func (a *API) GetManifestList(ctx context.Context, decryptor DecryptFunc, addr storage.Address, prefix string) (list ManifestList, err error) { apiManifestListCount.Inc(1) - walker, err := a.NewManifestWalker(ctx, addr, nil) + walker, err := a.NewManifestWalker(ctx, addr, decryptor, nil) if err != nil { apiManifestListFail.Inc(1) return ManifestList{}, err @@ -631,7 +690,7 @@ func (a *API) UpdateManifest(ctx context.Context, addr storage.Address, update f func (a *API) Modify(ctx context.Context, addr storage.Address, path, contentHash, contentType string) (storage.Address, error) { apiModifyCount.Inc(1) quitC := make(chan bool) - trie, err := loadManifest(ctx, a.fileStore, addr, quitC) + trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt) if err != nil { apiModifyFail.Inc(1) return nil, err @@ -663,7 +722,7 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content [] apiAddFileFail.Inc(1) return nil, "", err } - mkey, err := a.Resolve(ctx, uri) + mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) if err != nil { apiAddFileFail.Inc(1) return nil, "", err @@ -770,7 +829,7 @@ func (a *API) RemoveFile(ctx context.Context, mhash string, path string, fname s apiRmFileFail.Inc(1) return "", err } - mkey, err := a.Resolve(ctx, uri) + mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) if err != nil { apiRmFileFail.Inc(1) return "", err @@ -837,7 +896,7 @@ func (a *API) AppendFile(ctx context.Context, mhash, path, fname string, existin apiAppendFileFail.Inc(1) return nil, "", err } - mkey, err := a.Resolve(ctx, uri) + mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) if err != nil { apiAppendFileFail.Inc(1) return nil, "", err @@ -891,13 +950,13 @@ func (a *API) BuildDirectoryTree(ctx context.Context, mhash string, nameresolver if err != nil { return nil, nil, err } - addr, err = a.Resolve(ctx, uri) + addr, err = a.Resolve(ctx, uri.Addr) if err != nil { return nil, nil, err } quitC := make(chan bool) - rootTrie, err := loadManifest(ctx, a.fileStore, addr, quitC) + rootTrie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt) if err != nil { return nil, nil, fmt.Errorf("can't load manifest %v: %v", addr.String(), err) } @@ -955,7 +1014,7 @@ func (a *API) ResourceHashSize() int { // ResolveResourceManifest retrieves the Mutable Resource manifest for the given address, and returns the address of the metadata chunk. func (a *API) ResolveResourceManifest(ctx context.Context, addr storage.Address) (storage.Address, error) { - trie, err := loadManifest(ctx, a.fileStore, addr, nil) + trie, err := loadManifest(ctx, a.fileStore, addr, nil, NOOPDecrypt) if err != nil { return nil, fmt.Errorf("cannot load resource manifest: %v", err) } diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 78fab9508a..a65bf07e2e 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -19,6 +19,7 @@ package api import ( "context" "errors" + "flag" "fmt" "io" "io/ioutil" @@ -28,10 +29,17 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/storage" ) +func init() { + loglevel := flag.Int("loglevel", 2, "loglevel") + flag.Parse() + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) +} + func testAPI(t *testing.T, f func(*API, bool)) { datadir, err := ioutil.TempDir("", "bzz-test") if err != nil { @@ -42,7 +50,7 @@ func testAPI(t *testing.T, f func(*API, bool)) { if err != nil { return } - api := NewAPI(fileStore, nil, nil) + api := NewAPI(fileStore, nil, nil, nil) f(api, false) f(api, true) } @@ -85,7 +93,7 @@ func expResponse(content string, mimeType string, status int) *Response { func testGet(t *testing.T, api *API, bzzhash, path string) *testResponse { addr := storage.Address(common.Hex2Bytes(bzzhash)) - reader, mimeType, status, _, err := api.Get(context.TODO(), addr, path) + reader, mimeType, status, _, err := api.Get(context.TODO(), NOOPDecrypt, addr, path) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -229,7 +237,7 @@ func TestAPIResolve(t *testing.T) { if x.immutable { uri.Scheme = "bzz-immutable" } - res, err := api.Resolve(context.TODO(), uri) + res, err := api.ResolveURI(context.TODO(), uri, "") if err == nil { if x.expectErr != nil { t.Fatalf("expected error %q, got result %q", x.expectErr, res) @@ -373,3 +381,55 @@ func TestMultiResolver(t *testing.T) { }) } } + +func TestDecryptOriginForbidden(t *testing.T) { + ctx := context.TODO() + ctx = sctx.SetHost(ctx, "swarm-gateways.net") + + me := &ManifestEntry{ + Access: &AccessEntry{Type: AccessTypePass}, + } + + api := NewAPI(nil, nil, nil, nil) + + f := api.Decryptor(ctx, "") + err := f(me) + if err != ErrDecryptDomainForbidden { + t.Fatalf("should fail with ErrDecryptDomainForbidden, got %v", err) + } +} + +func TestDecryptOrigin(t *testing.T) { + for _, v := range []struct { + host string + expectError error + }{ + { + host: "localhost", + expectError: ErrDecrypt, + }, + { + host: "127.0.0.1", + expectError: ErrDecrypt, + }, + { + host: "swarm-gateways.net", + expectError: ErrDecryptDomainForbidden, + }, + } { + ctx := context.TODO() + ctx = sctx.SetHost(ctx, v.host) + + me := &ManifestEntry{ + Access: &AccessEntry{Type: AccessTypePass}, + } + + api := NewAPI(nil, nil, nil, nil) + + f := api.Decryptor(ctx, "") + err := f(me) + if err != v.expectError { + t.Fatalf("should fail with %v, got %v", v.expectError, err) + } + } +} diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index 8a9efe3608..3d06e9e1cc 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -43,6 +43,10 @@ var ( DefaultClient = NewClient(DefaultGateway) ) +var ( + ErrUnauthorized = errors.New("unauthorized") +) + func NewClient(gateway string) *Client { return &Client{ Gateway: gateway, @@ -188,7 +192,7 @@ func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bo // DownloadDirectory downloads the files contained in a swarm manifest under // the given path into a local directory (existing files will be overwritten) -func (c *Client) DownloadDirectory(hash, path, destDir string) error { +func (c *Client) DownloadDirectory(hash, path, destDir, credentials string) error { stat, err := os.Stat(destDir) if err != nil { return err @@ -201,13 +205,20 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error { if err != nil { return err } + if credentials != "" { + req.SetBasicAuth("", credentials) + } req.Header.Set("Accept", "application/x-tar") res, err := http.DefaultClient.Do(req) if err != nil { return err } defer res.Body.Close() - if res.StatusCode != http.StatusOK { + switch res.StatusCode { + case http.StatusOK: + case http.StatusUnauthorized: + return ErrUnauthorized + default: return fmt.Errorf("unexpected HTTP status: %s", res.Status) } tr := tar.NewReader(res.Body) @@ -248,7 +259,7 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error { // DownloadFile downloads a single file into the destination directory // if the manifest entry does not specify a file name - it will fallback // to the hash of the file as a filename -func (c *Client) DownloadFile(hash, path, dest string) error { +func (c *Client) DownloadFile(hash, path, dest, credentials string) error { hasDestinationFilename := false if stat, err := os.Stat(dest); err == nil { hasDestinationFilename = !stat.IsDir() @@ -261,9 +272,9 @@ func (c *Client) DownloadFile(hash, path, dest string) error { } } - manifestList, err := c.List(hash, path) + manifestList, err := c.List(hash, path, credentials) if err != nil { - return fmt.Errorf("could not list manifest: %v", err) + return err } switch len(manifestList.Entries) { @@ -280,13 +291,19 @@ func (c *Client) DownloadFile(hash, path, dest string) error { if err != nil { return err } + if credentials != "" { + req.SetBasicAuth("", credentials) + } res, err := http.DefaultClient.Do(req) if err != nil { return err } defer res.Body.Close() - - if res.StatusCode != http.StatusOK { + switch res.StatusCode { + case http.StatusOK: + case http.StatusUnauthorized: + return ErrUnauthorized + default: return fmt.Errorf("unexpected HTTP status: expected 200 OK, got %d", res.StatusCode) } filename := "" @@ -367,13 +384,24 @@ func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) { // - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt] // // where entries ending with "/" are common prefixes. -func (c *Client) List(hash, prefix string) (*api.ManifestList, error) { - res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix) +func (c *Client) List(hash, prefix, credentials string) (*api.ManifestList, error) { + req, err := http.NewRequest(http.MethodGet, c.Gateway+"/bzz-list:/"+hash+"/"+prefix, nil) + if err != nil { + return nil, err + } + if credentials != "" { + req.SetBasicAuth("", credentials) + } + res, err := http.DefaultClient.Do(req) if err != nil { return nil, err } defer res.Body.Close() - if res.StatusCode != http.StatusOK { + switch res.StatusCode { + case http.StatusOK: + case http.StatusUnauthorized: + return nil, ErrUnauthorized + default: return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) } var list api.ManifestList diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index ae82a91d79..2212f5c4c3 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -228,7 +228,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmp) - if err := client.DownloadDirectory(hash, "", tmp); err != nil { + if err := client.DownloadDirectory(hash, "", tmp, ""); err != nil { t.Fatal(err) } for _, file := range testDirFiles { @@ -265,7 +265,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) { } ls := func(prefix string) []string { - list, err := client.List(hash, prefix) + list, err := client.List(hash, prefix, "") if err != nil { t.Fatal(err) } diff --git a/swarm/api/encrypt.go b/swarm/api/encrypt.go new file mode 100644 index 0000000000..9a2e369149 --- /dev/null +++ b/swarm/api/encrypt.go @@ -0,0 +1,76 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "encoding/binary" + "errors" + + "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/swarm/storage/encryption" +) + +type RefEncryption struct { + spanEncryption encryption.Encryption + dataEncryption encryption.Encryption + span []byte +} + +func NewRefEncryption(refSize int) *RefEncryption { + span := make([]byte, 8) + binary.LittleEndian.PutUint64(span, uint64(refSize)) + return &RefEncryption{ + spanEncryption: encryption.New(0, uint32(refSize/32), sha3.NewKeccak256), + dataEncryption: encryption.New(refSize, 0, sha3.NewKeccak256), + span: span, + } +} + +func (re *RefEncryption) Encrypt(ref []byte, key []byte) ([]byte, error) { + encryptedSpan, err := re.spanEncryption.Encrypt(re.span, key) + if err != nil { + return nil, err + } + encryptedData, err := re.dataEncryption.Encrypt(ref, key) + if err != nil { + return nil, err + } + encryptedRef := make([]byte, len(ref)+8) + copy(encryptedRef[:8], encryptedSpan) + copy(encryptedRef[8:], encryptedData) + + return encryptedRef, nil +} + +func (re *RefEncryption) Decrypt(ref []byte, key []byte) ([]byte, error) { + decryptedSpan, err := re.spanEncryption.Decrypt(ref[:8], key) + if err != nil { + return nil, err + } + + size := binary.LittleEndian.Uint64(decryptedSpan) + if size != uint64(len(ref)-8) { + return nil, errors.New("invalid span in encrypted reference") + } + + decryptedRef, err := re.dataEncryption.Decrypt(ref[8:], key) + if err != nil { + return nil, err + } + + return decryptedRef, nil +} diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index aacd266998..8251ebc4dc 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -191,7 +191,7 @@ func (fs *FileSystem) Download(bzzpath, localpath string) error { if err != nil { return err } - addr, err := fs.api.Resolve(context.TODO(), uri) + addr, err := fs.api.Resolve(context.TODO(), uri.Addr) if err != nil { return err } @@ -202,7 +202,7 @@ func (fs *FileSystem) Download(bzzpath, localpath string) error { } quitC := make(chan bool) - trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC) + trie, err := loadManifest(context.TODO(), fs.api.fileStore, addr, quitC, NOOPDecrypt) if err != nil { log.Warn(fmt.Sprintf("fs.Download: loadManifestTrie error: %v", err)) return err diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index 84a2989d6e..fe7527b1f1 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -64,7 +64,7 @@ func TestApiDirUpload0(t *testing.T) { checkResponse(t, resp, exp) addr := storage.Address(common.Hex2Bytes(bzzhash)) - _, _, _, _, err = api.Get(context.TODO(), addr, "") + _, _, _, _, err = api.Get(context.TODO(), NOOPDecrypt, addr, "") if err == nil { t.Fatalf("expected error: %v", err) } @@ -143,7 +143,7 @@ func TestApiDirUploadModify(t *testing.T) { exp = expResponse(content, "text/css", 0) checkResponse(t, resp, exp) - _, _, _, _, err = api.Get(context.TODO(), addr, "") + _, _, _, _, err = api.Get(context.TODO(), nil, addr, "") if err == nil { t.Errorf("expected error: %v", err) } diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index c0d8d1a408..3b2dcc7d59 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -9,6 +9,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/sctx" "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/pborman/uuid" ) @@ -35,6 +36,15 @@ func SetRequestID(h http.Handler) http.Handler { }) } +func SetRequestHost(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r = r.WithContext(sctx.SetHost(r.Context(), r.Host)) + log.Info("setting request host", "ruid", GetRUID(r.Context()), "host", sctx.GetHost(r.Context())) + + h.ServeHTTP(w, r) + }) +} + func ParseURI(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) @@ -87,7 +97,7 @@ func RecoverPanic(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer func() { if err := recover(); err != nil { - log.Error("panic recovery!", "stack trace", debug.Stack(), "url", r.URL.String(), "headers", r.Header) + log.Error("panic recovery!", "stack trace", string(debug.Stack()), "url", r.URL.String(), "headers", r.Header) } }() h.ServeHTTP(w, r) diff --git a/swarm/api/http/response.go b/swarm/api/http/response.go index f050e706a8..c9fb9d2855 100644 --- a/swarm/api/http/response.go +++ b/swarm/api/http/response.go @@ -79,7 +79,7 @@ func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg s } func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) { - log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) + log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()), "code", code) RespondTemplate(w, r, "error", msg, code) } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 5a5c42adc0..b5ea0c23d4 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -23,7 +23,6 @@ import ( "bufio" "bytes" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -97,6 +96,7 @@ func NewServer(api *api.API, corsString string) *Server { defaultMiddlewares := []Adapter{ RecoverPanic, SetRequestID, + SetRequestHost, InitLoggingResponseWriter, ParseURI, InstrumentOpenTracing, @@ -169,6 +169,7 @@ func NewServer(api *api.API, corsString string) *Server { } func (s *Server) ListenAndServe(addr string) error { + s.listenAddr = addr return http.ListenAndServe(addr, s) } @@ -178,16 +179,24 @@ func (s *Server) ListenAndServe(addr string) error { // https://github.com/atom/electron/blob/master/docs/api/protocol.md type Server struct { http.Handler - api *api.API + api *api.API + listenAddr string } func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) { - log.Debug("handleBzzGet", "ruid", GetRUID(r.Context())) + log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()), "uri", r.RequestURI) if r.Header.Get("Accept") == "application/x-tar" { uri := GetURI(r.Context()) - reader, err := s.api.GetDirectoryTar(r.Context(), uri) + _, credentials, _ := r.BasicAuth() + reader, err := s.api.GetDirectoryTar(r.Context(), s.api.Decryptor(r.Context(), credentials), uri) if err != nil { + if isDecryptError(err) { + w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", uri.Address().String())) + RespondError(w, r, err.Error(), http.StatusUnauthorized) + return + } RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError) + return } defer reader.Close() @@ -287,7 +296,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) { var addr storage.Address if uri.Addr != "" && uri.Addr != "encrypt" { - addr, err = s.api.Resolve(r.Context(), uri) + addr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { postFilesFail.Inc(1) RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError) @@ -563,7 +572,7 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) { // resolve the content key. manifestAddr := uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(r.Context(), uri) + manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { getFail.Inc(1) RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) @@ -682,62 +691,21 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { uri := GetURI(r.Context()) log.Debug("handle.get", "ruid", ruid, "uri", uri) getCount.Inc(1) + _, pass, _ := r.BasicAuth() - var err error - addr := uri.Address() - if addr == nil { - addr, err = s.api.Resolve(r.Context(), uri) - if err != nil { - getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) - return - } - } else { - w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz:///path, so we are sure it is immutable. + addr, err := s.api.ResolveURI(r.Context(), uri, pass) + if err != nil { + getFail.Inc(1) + RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) + return } + w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz:///path, so we are sure it is immutable. log.Debug("handle.get: resolved", "ruid", ruid, "key", addr) // if path is set, interpret as a manifest and return the // raw entry at the given path - if uri.Path != "" { - walker, err := s.api.NewManifestWalker(r.Context(), addr, nil) - if err != nil { - getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest) - return - } - var entry *api.ManifestEntry - walker.Walk(func(e *api.ManifestEntry) error { - // if the entry matches the path, set entry and stop - // the walk - if e.Path == uri.Path { - entry = e - // return an error to cancel the walk - return errors.New("found") - } - // ignore non-manifest files - if e.ContentType != api.ManifestType { - return nil - } - - // if the manifest's path is a prefix of the - // requested path, recurse into it by returning - // nil and continuing the walk - if strings.HasPrefix(uri.Path, e.Path) { - return nil - } - - return api.ErrSkipManifest - }) - if entry == nil { - getFail.Inc(1) - RespondError(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound) - return - } - addr = storage.Address(common.Hex2Bytes(entry.Hash)) - } etag := common.Bytes2Hex(addr) noneMatchEtag := r.Header.Get("If-None-Match") w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key. @@ -781,6 +749,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) { func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { ruid := GetRUID(r.Context()) uri := GetURI(r.Context()) + _, credentials, _ := r.BasicAuth() log.Debug("handle.get.list", "ruid", ruid, "uri", uri) getListCount.Inc(1) @@ -790,7 +759,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { return } - addr, err := s.api.Resolve(r.Context(), uri) + addr, err := s.api.Resolve(r.Context(), uri.Addr) if err != nil { getListFail.Inc(1) RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) @@ -798,9 +767,14 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { } log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr) - list, err := s.api.GetManifestList(r.Context(), addr, uri.Path) + list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), addr, uri.Path) if err != nil { getListFail.Inc(1) + if isDecryptError(err) { + w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", addr.String())) + RespondError(w, r, err.Error(), http.StatusUnauthorized) + return + } RespondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -833,7 +807,8 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) { func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { ruid := GetRUID(r.Context()) uri := GetURI(r.Context()) - log.Debug("handle.get.file", "ruid", ruid) + _, credentials, _ := r.BasicAuth() + log.Debug("handle.get.file", "ruid", ruid, "uri", r.RequestURI) getFileCount.Inc(1) // ensure the root path has a trailing slash so that relative URLs work @@ -845,7 +820,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { manifestAddr := uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.Resolve(r.Context(), uri) + manifestAddr, err = s.api.ResolveURI(r.Context(), uri, credentials) if err != nil { getFileFail.Inc(1) RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) @@ -856,7 +831,8 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { } log.Debug("handle.get.file: resolved", "ruid", ruid, "key", manifestAddr) - reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, uri.Path) + + reader, contentType, status, contentKey, err := s.api.Get(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path) etag := common.Bytes2Hex(contentKey) noneMatchEtag := r.Header.Get("If-None-Match") @@ -869,6 +845,12 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { } if err != nil { + if isDecryptError(err) { + w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr)) + RespondError(w, r, err.Error(), http.StatusUnauthorized) + return + } + switch status { case http.StatusNotFound: getFileNotFound.Inc(1) @@ -883,9 +865,14 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { //the request results in ambiguous files //e.g. /read with readme.md and readinglist.txt available in manifest if status == http.StatusMultipleChoices { - list, err := s.api.GetManifestList(r.Context(), manifestAddr, uri.Path) + list, err := s.api.GetManifestList(r.Context(), s.api.Decryptor(r.Context(), credentials), manifestAddr, uri.Path) if err != nil { getFileFail.Inc(1) + if isDecryptError(err) { + w.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", manifestAddr)) + RespondError(w, r, err.Error(), http.StatusUnauthorized) + return + } RespondError(w, r, err.Error(), http.StatusInternalServerError) return } @@ -951,3 +938,7 @@ func (lrw *loggingResponseWriter) WriteHeader(code int) { lrw.statusCode = code lrw.ResponseWriter.WriteHeader(code) } + +func isDecryptError(err error) bool { + return strings.Contains(err.Error(), api.ErrDecrypt.Error()) +} diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 2a163dd39c..a1329a800f 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -46,13 +46,14 @@ type Manifest struct { // ManifestEntry represents an entry in a swarm manifest type ManifestEntry struct { - Hash string `json:"hash,omitempty"` - Path string `json:"path,omitempty"` - ContentType string `json:"contentType,omitempty"` - Mode int64 `json:"mode,omitempty"` - Size int64 `json:"size,omitempty"` - ModTime time.Time `json:"mod_time,omitempty"` - Status int `json:"status,omitempty"` + Hash string `json:"hash,omitempty"` + Path string `json:"path,omitempty"` + ContentType string `json:"contentType,omitempty"` + Mode int64 `json:"mode,omitempty"` + Size int64 `json:"size,omitempty"` + ModTime time.Time `json:"mod_time,omitempty"` + Status int `json:"status,omitempty"` + Access *AccessEntry `json:"access,omitempty"` } // ManifestList represents the result of listing files in a manifest @@ -98,7 +99,7 @@ type ManifestWriter struct { } func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWriter, error) { - trie, err := loadManifest(ctx, a.fileStore, addr, quitC) + trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt) if err != nil { return nil, fmt.Errorf("error loading manifest %s: %s", addr, err) } @@ -141,8 +142,8 @@ type ManifestWalker struct { quitC chan bool } -func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWalker, error) { - trie, err := loadManifest(ctx, a.fileStore, addr, quitC) +func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, decrypt DecryptFunc, quitC chan bool) (*ManifestWalker, error) { + trie, err := loadManifest(ctx, a.fileStore, addr, quitC, decrypt) if err != nil { return nil, fmt.Errorf("error loading manifest %s: %s", addr, err) } @@ -194,6 +195,7 @@ type manifestTrie struct { entries [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry ref storage.Address // if ref != nil, it is stored encrypted bool + decrypt DecryptFunc } func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry { @@ -209,15 +211,15 @@ type manifestTrieEntry struct { subtrie *manifestTrie } -func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand log.Trace("manifest lookup", "key", hash) // retrieve manifest via FileStore manifestReader, isEncrypted := fileStore.Retrieve(ctx, hash) log.Trace("reader retrieved", "key", hash) - return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC) + return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC, decrypt) } -func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand // TODO check size for oversized manifests size, err := mr.Size(mr.Context(), quitC) @@ -258,26 +260,41 @@ func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore trie = &manifestTrie{ fileStore: fileStore, encrypted: isEncrypted, + decrypt: decrypt, } for _, entry := range man.Entries { - trie.addEntry(entry, quitC) + err = trie.addEntry(entry, quitC) + if err != nil { + return + } } return } -func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { +func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) error { mt.ref = nil // trie modified, hash needs to be re-calculated on demand + if entry.ManifestEntry.Access != nil { + if mt.decrypt == nil { + return errors.New("dont have decryptor") + } + + err := mt.decrypt(&entry.ManifestEntry) + if err != nil { + return err + } + } + if len(entry.Path) == 0 { mt.entries[256] = entry - return + return nil } b := entry.Path[0] oldentry := mt.entries[b] if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) { mt.entries[b] = entry - return + return nil } cpl := 0 @@ -287,12 +304,12 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) { if mt.loadSubTrie(oldentry, quitC) != nil { - return + return nil } entry.Path = entry.Path[cpl:] oldentry.subtrie.addEntry(entry, quitC) oldentry.Hash = "" - return + return nil } commonPrefix := entry.Path[:cpl] @@ -310,6 +327,7 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { Path: commonPrefix, ContentType: ManifestType, }, subtrie) + return nil } func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { @@ -398,9 +416,20 @@ func (mt *manifestTrie) recalcAndStore() error { } func (mt *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) { + if entry.ManifestEntry.Access != nil { + if mt.decrypt == nil { + return errors.New("dont have decryptor") + } + + err := mt.decrypt(&entry.ManifestEntry) + if err != nil { + return err + } + } + if entry.subtrie == nil { hash := common.Hex2Bytes(entry.Hash) - entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC) + entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC, mt.decrypt) entry.Hash = "" // might not match, should be recalculated } return diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go index d65f023f85..1c8e53c433 100644 --- a/swarm/api/manifest_test.go +++ b/swarm/api/manifest_test.go @@ -44,7 +44,7 @@ func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...stri quitC := make(chan bool) fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams()) ref := make([]byte, fileStore.HashSize()) - trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC) + trie, err := readManifest(manifest(paths...), ref, fileStore, false, quitC, NOOPDecrypt) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } @@ -101,7 +101,7 @@ func TestExactMatch(t *testing.T) { mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map") fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams()) ref := make([]byte, fileStore.HashSize()) - trie, err := readManifest(mf, ref, fileStore, false, quitC) + trie, err := readManifest(mf, ref, fileStore, false, quitC, nil) if err != nil { t.Errorf("unexpected error making manifest: %v", err) } @@ -134,7 +134,7 @@ func TestAddFileWithManifestPath(t *testing.T) { } fileStore := storage.NewFileStore(nil, storage.NewFileStoreParams()) ref := make([]byte, fileStore.HashSize()) - trie, err := readManifest(reader, ref, fileStore, false, nil) + trie, err := readManifest(reader, ref, fileStore, false, nil, NOOPDecrypt) if err != nil { t.Fatal(err) } @@ -161,7 +161,7 @@ func TestReadManifestOverSizeLimit(t *testing.T) { reader := &storage.LazyTestSectionReader{ SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))), } - _, err := readManifest(reader, storage.Address{}, nil, false, nil) + _, err := readManifest(reader, storage.Address{}, nil, false, nil, NOOPDecrypt) if err == nil { t.Fatal("got no error from readManifest") } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 3b52301a01..8a48fe5bc0 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -63,11 +63,11 @@ func (s *Storage) Get(ctx context.Context, bzzpath string) (*Response, error) { if err != nil { return nil, err } - addr, err := s.api.Resolve(ctx, uri) + addr, err := s.api.Resolve(ctx, uri.Addr) if err != nil { return nil, err } - reader, mimeType, status, _, err := s.api.Get(ctx, addr, uri.Path) + reader, mimeType, status, _, err := s.api.Get(ctx, nil, addr, uri.Path) if err != nil { return nil, err } @@ -93,7 +93,7 @@ func (s *Storage) Modify(ctx context.Context, rootHash, path, contentHash, conte if err != nil { return "", err } - addr, err := s.api.Resolve(ctx, uri) + addr, err := s.api.Resolve(ctx, uri.Addr) if err != nil { return "", err } diff --git a/swarm/api/uri.go b/swarm/api/uri.go index 14965e0d93..808517088a 100644 --- a/swarm/api/uri.go +++ b/swarm/api/uri.go @@ -53,6 +53,19 @@ type URI struct { Path string } +func (u *URI) MarshalJSON() (out []byte, err error) { + return []byte(`"` + u.String() + `"`), nil +} + +func (u *URI) UnmarshalJSON(value []byte) error { + uri, err := Parse(string(value)) + if err != nil { + return err + } + *u = *uri + return nil +} + // Parse parses rawuri into a URI struct, where rawuri is expected to have one // of the following formats: // diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index d579d15a02..6efeb78d9b 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -1650,7 +1650,7 @@ func TestFUSE(t *testing.T) { if err != nil { t.Fatal(err) } - ta := &testAPI{api: api.NewAPI(fileStore, nil, nil)} + ta := &testAPI{api: api.NewAPI(fileStore, nil, nil, nil)} //run a short suite of tests //approx time: 28s diff --git a/swarm/network_test.go b/swarm/network_test.go index d2a0309338..176c635d82 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -445,7 +445,7 @@ func retrieve( log.Debug("api get: check file", "node", id.String(), "key", f.addr.String(), "total files found", atomic.LoadUint64(totalFoundCount)) - r, _, _, _, err := swarm.api.Get(context.TODO(), f.addr, "/") + r, _, _, _, err := swarm.api.Get(context.TODO(), api.NOOPDecrypt, f.addr, "/") if err != nil { errc <- fmt.Errorf("api get: node %s, key %s, kademlia %s: %v", id, f.addr, swarm.bzz.Hive, err) return diff --git a/swarm/sctx/sctx.go b/swarm/sctx/sctx.go index 8619f6e19c..bed2b11458 100644 --- a/swarm/sctx/sctx.go +++ b/swarm/sctx/sctx.go @@ -1,7 +1,22 @@ package sctx +import "context" + type ContextKey int const ( HTTPRequestIDKey ContextKey = iota + requestHostKey ) + +func SetHost(ctx context.Context, domain string) context.Context { + return context.WithValue(ctx, requestHostKey, domain) +} + +func GetHost(ctx context.Context) string { + v, ok := ctx.Value(requestHostKey).(string) + if ok { + return v + } + return "" +} diff --git a/swarm/swarm.go b/swarm/swarm.go index f731ff33d7..a895bdfa55 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -85,14 +85,12 @@ type Swarm struct { type SwarmAPI struct { Api *api.API Backend chequebook.Backend - PrvKey *ecdsa.PrivateKey } func (self *Swarm) API() *SwarmAPI { return &SwarmAPI{ Api: self.api, Backend: self.backend, - PrvKey: self.privateKey, } } @@ -217,7 +215,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e pss.SetHandshakeController(self.ps, pss.NewHandshakeParams()) } - self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler) + self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler, self.privateKey) // Manifests for Smart Hosting log.Debug(fmt.Sprintf("-> Web3 virtual server API")) diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 238f783088..7fd60fcc3d 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -77,7 +77,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer) *Tes t.Fatal(err) } - a := api.NewAPI(fileStore, nil, rh.Handler) + a := api.NewAPI(fileStore, nil, rh.Handler, nil) srv := httptest.NewServer(serverFunc(a)) return &TestSwarmServer{ Server: srv, From 2cdf6ee7e00d6127c372e7a28bb27a80ef495cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Wed, 15 Aug 2018 22:25:46 +0200 Subject: [PATCH 14/96] light: CHT and bloom trie indexers working in light mode (#16534) This PR enables the indexers to work in light client mode by downloading a part of these tries (the Merkle proofs of the last values of the last known section) in order to be able to add new values and recalculate subsequent hashes. It also adds CHT data to NodeInfo. --- core/chain_indexer.go | 30 +++++-- core/chain_indexer_test.go | 6 +- eth/backend.go | 2 +- eth/bloombits.go | 23 +++-- les/backend.go | 39 +++++---- les/distributor.go | 4 - les/handler.go | 34 ++++++-- les/helper_test.go | 6 +- les/odr.go | 18 ++-- les/odr_test.go | 3 +- les/request_test.go | 3 +- les/retrieve.go | 3 +- les/server.go | 4 +- light/lightchain.go | 14 +-- light/odr.go | 4 + light/postprocess.go | 170 ++++++++++++++++++++++++++++--------- 16 files changed, 251 insertions(+), 112 deletions(-) diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 0b927116d0..11a7c96fa0 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -17,6 +17,7 @@ package core import ( + "context" "encoding/binary" "fmt" "sync" @@ -37,11 +38,11 @@ import ( type ChainIndexerBackend interface { // Reset initiates the processing of a new chain segment, potentially terminating // any partially completed operations (in case of a reorg). - Reset(section uint64, prevHead common.Hash) error + Reset(ctx context.Context, section uint64, prevHead common.Hash) error // Process crunches through the next header in the chain segment. The caller // will ensure a sequential order of headers. - Process(header *types.Header) + Process(ctx context.Context, header *types.Header) error // Commit finalizes the section metadata and stores it into the database. Commit() error @@ -71,9 +72,11 @@ type ChainIndexer struct { backend ChainIndexerBackend // Background processor generating the index data content children []*ChainIndexer // Child indexers to cascade chain updates to - active uint32 // Flag whether the event loop was started - update chan struct{} // Notification channel that headers should be processed - quit chan chan error // Quit channel to tear down running goroutines + active uint32 // Flag whether the event loop was started + update chan struct{} // Notification channel that headers should be processed + quit chan chan error // Quit channel to tear down running goroutines + ctx context.Context + ctxCancel func() sectionSize uint64 // Number of blocks in a single chain segment to process confirmsReq uint64 // Number of confirmations before processing a completed segment @@ -105,6 +108,8 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken } // Initialize database dependent fields and start the updater c.loadValidSections() + c.ctx, c.ctxCancel = context.WithCancel(context.Background()) + go c.updateLoop() return c @@ -138,6 +143,8 @@ func (c *ChainIndexer) Start(chain ChainIndexerChain) { func (c *ChainIndexer) Close() error { var errs []error + c.ctxCancel() + // Tear down the primary update loop errc := make(chan error) c.quit <- errc @@ -297,6 +304,12 @@ func (c *ChainIndexer) updateLoop() { c.lock.Unlock() newHead, err := c.processSection(section, oldHead) if err != nil { + select { + case <-c.ctx.Done(): + <-c.quit <- nil + return + default: + } c.log.Error("Section processing failed", "error", err) } c.lock.Lock() @@ -344,7 +357,7 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com // Reset and partial processing - if err := c.backend.Reset(section, lastHead); err != nil { + if err := c.backend.Reset(c.ctx, section, lastHead); err != nil { c.setValidSections(0) return common.Hash{}, err } @@ -360,11 +373,12 @@ func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (com } else if header.ParentHash != lastHead { return common.Hash{}, fmt.Errorf("chain reorged during section processing") } - c.backend.Process(header) + if err := c.backend.Process(c.ctx, header); err != nil { + return common.Hash{}, err + } lastHead = header.Hash() } if err := c.backend.Commit(); err != nil { - c.log.Error("Section commit failed", "error", err) return common.Hash{}, err } return lastHead, nil diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index 550caf5567..a029dec626 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -17,6 +17,7 @@ package core import ( + "context" "fmt" "math/big" "math/rand" @@ -210,13 +211,13 @@ func (b *testChainIndexBackend) reorg(headNum uint64) uint64 { return b.stored * b.indexer.sectionSize } -func (b *testChainIndexBackend) Reset(section uint64, prevHead common.Hash) error { +func (b *testChainIndexBackend) Reset(ctx context.Context, section uint64, prevHead common.Hash) error { b.section = section b.headerCnt = 0 return nil } -func (b *testChainIndexBackend) Process(header *types.Header) { +func (b *testChainIndexBackend) Process(ctx context.Context, header *types.Header) error { b.headerCnt++ if b.headerCnt > b.indexer.sectionSize { b.t.Error("Processing too many headers") @@ -227,6 +228,7 @@ func (b *testChainIndexBackend) Process(header *types.Header) { b.t.Fatal("Unexpected call to Process") case b.processCh <- header.Number.Uint64(): } + return nil } func (b *testChainIndexBackend) Commit() error { diff --git a/eth/backend.go b/eth/backend.go index 865534b198..6549cb8a3d 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -130,7 +130,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { gasPrice: config.GasPrice, etherbase: config.Etherbase, bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks), + bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, bloomConfirms), } log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) diff --git a/eth/bloombits.go b/eth/bloombits.go index 954239d141..eb18565e2c 100644 --- a/eth/bloombits.go +++ b/eth/bloombits.go @@ -17,6 +17,7 @@ package eth import ( + "context" "time" "github.com/ethereum/go-ethereum/common" @@ -92,30 +93,28 @@ const ( // BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index // for the Ethereum header bloom filters, permitting blazing fast filtering. type BloomIndexer struct { - size uint64 // section size to generate bloombits for - - db ethdb.Database // database instance to write index data and metadata into - gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index - - section uint64 // Section is the section number being processed currently - head common.Hash // Head is the hash of the last header processed + size uint64 // section size to generate bloombits for + db ethdb.Database // database instance to write index data and metadata into + gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index + section uint64 // Section is the section number being processed currently + head common.Hash // Head is the hash of the last header processed } // NewBloomIndexer returns a chain indexer that generates bloom bits data for the // canonical chain for fast logs filtering. -func NewBloomIndexer(db ethdb.Database, size uint64) *core.ChainIndexer { +func NewBloomIndexer(db ethdb.Database, size, confReq uint64) *core.ChainIndexer { backend := &BloomIndexer{ db: db, size: size, } table := ethdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) - return core.NewChainIndexer(db, table, backend, size, bloomConfirms, bloomThrottling, "bloombits") + return core.NewChainIndexer(db, table, backend, size, confReq, bloomThrottling, "bloombits") } // Reset implements core.ChainIndexerBackend, starting a new bloombits index // section. -func (b *BloomIndexer) Reset(section uint64, lastSectionHead common.Hash) error { +func (b *BloomIndexer) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { gen, err := bloombits.NewGenerator(uint(b.size)) b.gen, b.section, b.head = gen, section, common.Hash{} return err @@ -123,16 +122,16 @@ func (b *BloomIndexer) Reset(section uint64, lastSectionHead common.Hash) error // Process implements core.ChainIndexerBackend, adding a new header's bloom into // the index. -func (b *BloomIndexer) Process(header *types.Header) { +func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error { b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom) b.head = header.Hash() + return nil } // Commit implements core.ChainIndexerBackend, finalizing the bloom section and // writing it out into the database. func (b *BloomIndexer) Commit() error { batch := b.db.NewBatch() - for i := 0; i < types.BloomBitLength; i++ { bits, err := b.gen.Bitset(uint(i)) if err != nil { diff --git a/les/backend.go b/les/backend.go index 178bc1e0e4..9b8cc1828f 100644 --- a/les/backend.go +++ b/les/backend.go @@ -95,29 +95,35 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { quitSync := make(chan struct{}) leth := &LightEthereum{ - config: config, - chainConfig: chainConfig, - chainDb: chainDb, - eventMux: ctx.EventMux, - peers: peers, - reqDist: newRequestDistributor(peers, quitSync), - accountManager: ctx.AccountManager, - engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, chainDb), - shutdownChan: make(chan bool), - networkId: config.NetworkId, - bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: eth.NewBloomIndexer(chainDb, light.BloomTrieFrequency), - chtIndexer: light.NewChtIndexer(chainDb, true), - bloomTrieIndexer: light.NewBloomTrieIndexer(chainDb, true), + config: config, + chainConfig: chainConfig, + chainDb: chainDb, + eventMux: ctx.EventMux, + peers: peers, + reqDist: newRequestDistributor(peers, quitSync), + accountManager: ctx.AccountManager, + engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, chainDb), + shutdownChan: make(chan bool), + networkId: config.NetworkId, + bloomRequests: make(chan chan *bloombits.Retrieval), + bloomIndexer: eth.NewBloomIndexer(chainDb, light.BloomTrieFrequency, light.HelperTrieConfirmations), } leth.relay = NewLesTxRelay(peers, leth.reqDist) leth.serverPool = newServerPool(chainDb, quitSync, &leth.wg) leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool) - leth.odr = NewLesOdr(chainDb, leth.chtIndexer, leth.bloomTrieIndexer, leth.bloomIndexer, leth.retriever) + leth.odr = NewLesOdr(chainDb, leth.retriever) + leth.chtIndexer = light.NewChtIndexer(chainDb, true, leth.odr) + leth.bloomTrieIndexer = light.NewBloomTrieIndexer(chainDb, true, leth.odr) + leth.odr.SetIndexers(leth.chtIndexer, leth.bloomTrieIndexer, leth.bloomIndexer) + // Note: NewLightChain adds the trusted checkpoint so it needs an ODR with + // indexers already set but not started yet if leth.blockchain, err = light.NewLightChain(leth.odr, leth.chainConfig, leth.engine); err != nil { return nil, err } + // Note: AddChildIndexer starts the update process for the child + leth.bloomIndexer.AddChildIndexer(leth.bloomTrieIndexer) + leth.chtIndexer.Start(leth.blockchain) leth.bloomIndexer.Start(leth.blockchain) // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { @@ -242,9 +248,6 @@ func (s *LightEthereum) Stop() error { if s.chtIndexer != nil { s.chtIndexer.Close() } - if s.bloomTrieIndexer != nil { - s.bloomTrieIndexer.Close() - } s.blockchain.Stop() s.protocolManager.Stop() s.txPool.Stop() diff --git a/les/distributor.go b/les/distributor.go index 159fa4c73f..d3f6b21d18 100644 --- a/les/distributor.go +++ b/les/distributor.go @@ -20,14 +20,10 @@ package les import ( "container/list" - "errors" "sync" "time" ) -// ErrNoPeers is returned if no peers capable of serving a queued request are available -var ErrNoPeers = errors.New("no suitable peers available") - // requestDistributor implements a mechanism that distributes requests to // suitable peers, obeying flow control rules and prioritizing them in creation // order (even when a resend is necessary). diff --git a/les/handler.go b/les/handler.go index 91a235bf0a..ccb4a88448 100644 --- a/les/handler.go +++ b/les/handler.go @@ -1206,11 +1206,12 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { // NodeInfo represents a short summary of the Ethereum sub-protocol metadata // known about the host peer. type NodeInfo struct { - Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) - Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain - Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block - Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules - Head common.Hash `json:"head"` // SHA3 hash of the host's best owned block + Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) + Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain + Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block + Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules + Head common.Hash `json:"head"` // SHA3 hash of the host's best owned block + CHT light.TrustedCheckpoint `json:"cht"` // Trused CHT checkpoint for fast catchup } // NodeInfo retrieves some protocol metadata about the running host node. @@ -1218,12 +1219,31 @@ func (self *ProtocolManager) NodeInfo() *NodeInfo { head := self.blockchain.CurrentHeader() hash := head.Hash() + var cht light.TrustedCheckpoint + + sections, _, sectionHead := self.odr.ChtIndexer().Sections() + sections2, _, sectionHead2 := self.odr.BloomTrieIndexer().Sections() + if sections2 < sections { + sections = sections2 + sectionHead = sectionHead2 + } + if sections > 0 { + sectionIndex := sections - 1 + cht = light.TrustedCheckpoint{ + SectionIdx: sectionIndex, + SectionHead: sectionHead, + CHTRoot: light.GetChtRoot(self.chainDb, sectionIndex, sectionHead), + BloomRoot: light.GetBloomTrieRoot(self.chainDb, sectionIndex, sectionHead), + } + } + return &NodeInfo{ Network: self.networkId, Difficulty: self.blockchain.GetTd(hash, head.Number.Uint64()), Genesis: self.blockchain.Genesis().Hash(), Config: self.blockchain.Config(), Head: hash, + CHT: cht, } } @@ -1258,7 +1278,7 @@ func (pc *peerConnection) RequestHeadersByHash(origin common.Hash, amount int, s } _, ok := <-pc.manager.reqDist.queue(rq) if !ok { - return ErrNoPeers + return light.ErrNoPeers } return nil } @@ -1282,7 +1302,7 @@ func (pc *peerConnection) RequestHeadersByNumber(origin uint64, amount int, skip } _, ok := <-pc.manager.reqDist.queue(rq) if !ok { - return ErrNoPeers + return light.ErrNoPeers } return nil } diff --git a/les/helper_test.go b/les/helper_test.go index 8fd01a39e0..50c97e06e1 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -156,12 +156,12 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor } else { blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) - chtIndexer := light.NewChtIndexer(db, false) + chtIndexer := light.NewChtIndexer(db, false, nil) chtIndexer.Start(blockchain) - bbtIndexer := light.NewBloomTrieIndexer(db, false) + bbtIndexer := light.NewBloomTrieIndexer(db, false, nil) - bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks) + bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks, light.HelperTrieProcessConfirmations) bloomIndexer.AddChildIndexer(bbtIndexer) bloomIndexer.Start(blockchain) diff --git a/les/odr.go b/les/odr.go index f8412aaad7..2ad28d5d9f 100644 --- a/les/odr.go +++ b/les/odr.go @@ -33,14 +33,11 @@ type LesOdr struct { stop chan struct{} } -func NewLesOdr(db ethdb.Database, chtIndexer, bloomTrieIndexer, bloomIndexer *core.ChainIndexer, retriever *retrieveManager) *LesOdr { +func NewLesOdr(db ethdb.Database, retriever *retrieveManager) *LesOdr { return &LesOdr{ - db: db, - chtIndexer: chtIndexer, - bloomTrieIndexer: bloomTrieIndexer, - bloomIndexer: bloomIndexer, - retriever: retriever, - stop: make(chan struct{}), + db: db, + retriever: retriever, + stop: make(chan struct{}), } } @@ -54,6 +51,13 @@ func (odr *LesOdr) Database() ethdb.Database { return odr.db } +// SetIndexers adds the necessary chain indexers to the ODR backend +func (odr *LesOdr) SetIndexers(chtIndexer, bloomTrieIndexer, bloomIndexer *core.ChainIndexer) { + odr.chtIndexer = chtIndexer + odr.bloomTrieIndexer = bloomTrieIndexer + odr.bloomIndexer = bloomIndexer +} + // ChtIndexer returns the CHT chain indexer func (odr *LesOdr) ChtIndexer() *core.ChainIndexer { return odr.chtIndexer diff --git a/les/odr_test.go b/les/odr_test.go index 983f7262b0..c7c25cbe4b 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -167,7 +167,8 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { rm := newRetrieveManager(peers, dist, nil) db := ethdb.NewMemDatabase() ldb := ethdb.NewMemDatabase() - odr := NewLesOdr(ldb, light.NewChtIndexer(db, true), light.NewBloomTrieIndexer(db, true), eth.NewBloomIndexer(db, light.BloomTrieFrequency), rm) + odr := NewLesOdr(ldb, rm) + odr.SetIndexers(light.NewChtIndexer(db, true, nil), light.NewBloomTrieIndexer(db, true, nil), eth.NewBloomIndexer(db, light.BloomTrieFrequency, light.HelperTrieConfirmations)) pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) _, err1, lpeer, err2 := newTestPeerPair("peer", protocol, pm, lpm) diff --git a/les/request_test.go b/les/request_test.go index ba2f603d8b..db576798b4 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -89,7 +89,8 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { rm := newRetrieveManager(peers, dist, nil) db := ethdb.NewMemDatabase() ldb := ethdb.NewMemDatabase() - odr := NewLesOdr(ldb, light.NewChtIndexer(db, true), light.NewBloomTrieIndexer(db, true), eth.NewBloomIndexer(db, light.BloomTrieFrequency), rm) + odr := NewLesOdr(ldb, rm) + odr.SetIndexers(light.NewChtIndexer(db, true, nil), light.NewBloomTrieIndexer(db, true, nil), eth.NewBloomIndexer(db, light.BloomTrieFrequency, light.HelperTrieConfirmations)) pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) diff --git a/les/retrieve.go b/les/retrieve.go index a9037a38ef..8ae36d82cd 100644 --- a/les/retrieve.go +++ b/les/retrieve.go @@ -27,6 +27,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/light" ) var ( @@ -207,7 +208,7 @@ func (r *sentReq) stateRequesting() reqStateFn { return r.stateNoMorePeers } // nothing to wait for, no more peers to ask, return with error - r.stop(ErrNoPeers) + r.stop(light.ErrNoPeers) // no need to go to stopped state because waiting() already returned false return nil } diff --git a/les/server.go b/les/server.go index fca6124c9c..a934fbf26e 100644 --- a/les/server.go +++ b/les/server.go @@ -67,8 +67,8 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { protocolManager: pm, quitSync: quitSync, lesTopics: lesTopics, - chtIndexer: light.NewChtIndexer(eth.ChainDb(), false), - bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), false), + chtIndexer: light.NewChtIndexer(eth.ChainDb(), false, nil), + bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), false, nil), } logger := log.New() diff --git a/light/lightchain.go b/light/lightchain.go index 30b9bd89a6..b7e629e88b 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -116,19 +116,19 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. } // addTrustedCheckpoint adds a trusted checkpoint to the blockchain -func (self *LightChain) addTrustedCheckpoint(cp trustedCheckpoint) { +func (self *LightChain) addTrustedCheckpoint(cp TrustedCheckpoint) { if self.odr.ChtIndexer() != nil { - StoreChtRoot(self.chainDb, cp.sectionIdx, cp.sectionHead, cp.chtRoot) - self.odr.ChtIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead) + StoreChtRoot(self.chainDb, cp.SectionIdx, cp.SectionHead, cp.CHTRoot) + self.odr.ChtIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) } if self.odr.BloomTrieIndexer() != nil { - StoreBloomTrieRoot(self.chainDb, cp.sectionIdx, cp.sectionHead, cp.bloomTrieRoot) - self.odr.BloomTrieIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead) + StoreBloomTrieRoot(self.chainDb, cp.SectionIdx, cp.SectionHead, cp.BloomRoot) + self.odr.BloomTrieIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) } if self.odr.BloomIndexer() != nil { - self.odr.BloomIndexer().AddKnownSectionHead(cp.sectionIdx, cp.sectionHead) + self.odr.BloomIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) } - log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.sectionIdx+1)*CHTFrequencyClient-1, "hash", cp.sectionHead) + log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.SectionIdx+1)*CHTFrequencyClient-1, "hash", cp.SectionHead) } func (self *LightChain) getProcInterrupt() bool { diff --git a/light/odr.go b/light/odr.go index 8f1e50b817..83c64055a2 100644 --- a/light/odr.go +++ b/light/odr.go @@ -20,6 +20,7 @@ package light import ( "context" + "errors" "math/big" "github.com/ethereum/go-ethereum/common" @@ -33,6 +34,9 @@ import ( // service is not required. var NoOdr = context.Background() +// ErrNoPeers is returned if no peers capable of serving a queued request are available +var ErrNoPeers = errors.New("no suitable peers available") + // OdrBackend is an interface to a backend service that handles ODR retrievals type type OdrBackend interface { Database() ethdb.Database diff --git a/light/postprocess.go b/light/postprocess.go index 2090a9d044..0b25e1d881 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -17,8 +17,10 @@ package light import ( + "context" "encoding/binary" "errors" + "fmt" "math/big" "time" @@ -47,35 +49,35 @@ const ( HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated ) -// trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with +// TrustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with // the appropriate section index and head hash. It is used to start light syncing from this checkpoint // and avoid downloading the entire header chain while still being able to securely access old headers/logs. -type trustedCheckpoint struct { - name string - sectionIdx uint64 - sectionHead, chtRoot, bloomTrieRoot common.Hash +type TrustedCheckpoint struct { + name string + SectionIdx uint64 + SectionHead, CHTRoot, BloomRoot common.Hash } var ( - mainnetCheckpoint = trustedCheckpoint{ - name: "mainnet", - sectionIdx: 179, - sectionHead: common.HexToHash("ae778e455492db1183e566fa0c67f954d256fdd08618f6d5a393b0e24576d0ea"), - chtRoot: common.HexToHash("646b338f9ca74d936225338916be53710ec84020b89946004a8605f04c817f16"), - bloomTrieRoot: common.HexToHash("d0f978f5dbc86e5bf931d8dd5b2ecbebbda6dc78f8896af6a27b46a3ced0ac25"), + mainnetCheckpoint = TrustedCheckpoint{ + name: "mainnet", + SectionIdx: 179, + SectionHead: common.HexToHash("ae778e455492db1183e566fa0c67f954d256fdd08618f6d5a393b0e24576d0ea"), + CHTRoot: common.HexToHash("646b338f9ca74d936225338916be53710ec84020b89946004a8605f04c817f16"), + BloomRoot: common.HexToHash("d0f978f5dbc86e5bf931d8dd5b2ecbebbda6dc78f8896af6a27b46a3ced0ac25"), } - ropstenCheckpoint = trustedCheckpoint{ - name: "ropsten", - sectionIdx: 107, - sectionHead: common.HexToHash("e1988f95399debf45b873e065e5cd61b416ef2e2e5deec5a6f87c3127086e1ce"), - chtRoot: common.HexToHash("15cba18e4de0ab1e95e202625199ba30147aec8b0b70384b66ebea31ba6a18e0"), - bloomTrieRoot: common.HexToHash("e00fa6389b2e597d9df52172cd8e936879eed0fca4fa59db99e2c8ed682562f2"), + ropstenCheckpoint = TrustedCheckpoint{ + name: "ropsten", + SectionIdx: 107, + SectionHead: common.HexToHash("e1988f95399debf45b873e065e5cd61b416ef2e2e5deec5a6f87c3127086e1ce"), + CHTRoot: common.HexToHash("15cba18e4de0ab1e95e202625199ba30147aec8b0b70384b66ebea31ba6a18e0"), + BloomRoot: common.HexToHash("e00fa6389b2e597d9df52172cd8e936879eed0fca4fa59db99e2c8ed682562f2"), } ) // trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to -var trustedCheckpoints = map[common.Hash]trustedCheckpoint{ +var trustedCheckpoints = map[common.Hash]TrustedCheckpoint{ params.MainnetGenesisHash: mainnetCheckpoint, params.TestnetGenesisHash: ropstenCheckpoint, } @@ -119,7 +121,8 @@ func StoreChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common // ChtIndexerBackend implements core.ChainIndexerBackend type ChtIndexerBackend struct { - diskdb ethdb.Database + diskdb, trieTable ethdb.Database + odr OdrBackend triedb *trie.Database section, sectionSize uint64 lastHash common.Hash @@ -127,7 +130,7 @@ type ChtIndexerBackend struct { } // NewBloomTrieIndexer creates a BloomTrie chain indexer -func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { +func NewChtIndexer(db ethdb.Database, clientMode bool, odr OdrBackend) *core.ChainIndexer { var sectionSize, confirmReq uint64 if clientMode { sectionSize = CHTFrequencyClient @@ -137,28 +140,64 @@ func NewChtIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { confirmReq = HelperTrieProcessConfirmations } idb := ethdb.NewTable(db, "chtIndex-") + trieTable := ethdb.NewTable(db, ChtTablePrefix) backend := &ChtIndexerBackend{ diskdb: db, - triedb: trie.NewDatabase(ethdb.NewTable(db, ChtTablePrefix)), + odr: odr, + trieTable: trieTable, + triedb: trie.NewDatabase(trieTable), sectionSize: sectionSize, } return core.NewChainIndexer(db, idb, backend, sectionSize, confirmReq, time.Millisecond*100, "cht") } +// fetchMissingNodes tries to retrieve the last entry of the latest trusted CHT from the +// ODR backend in order to be able to add new entries and calculate subsequent root hashes +func (c *ChtIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error { + batch := c.trieTable.NewBatch() + r := &ChtRequest{ChtRoot: root, ChtNum: section - 1, BlockNum: section*c.sectionSize - 1} + for { + err := c.odr.Retrieve(ctx, r) + switch err { + case nil: + r.Proof.Store(batch) + return batch.Write() + case ErrNoPeers: + // if there are no peers to serve, retry later + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second * 10): + // stay in the loop and try again + } + default: + return err + } + } +} + // Reset implements core.ChainIndexerBackend -func (c *ChtIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { +func (c *ChtIndexerBackend) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { var root common.Hash if section > 0 { root = GetChtRoot(c.diskdb, section-1, lastSectionHead) } var err error c.trie, err = trie.New(root, c.triedb) + + if err != nil && c.odr != nil { + err = c.fetchMissingNodes(ctx, section, root) + if err == nil { + c.trie, err = trie.New(root, c.triedb) + } + } + c.section = section return err } // Process implements core.ChainIndexerBackend -func (c *ChtIndexerBackend) Process(header *types.Header) { +func (c *ChtIndexerBackend) Process(ctx context.Context, header *types.Header) error { hash, num := header.Hash(), header.Number.Uint64() c.lastHash = hash @@ -170,6 +209,7 @@ func (c *ChtIndexerBackend) Process(header *types.Header) { binary.BigEndian.PutUint64(encNumber[:], num) data, _ := rlp.EncodeToBytes(ChtNode{hash, td}) c.trie.Update(encNumber[:], data) + return nil } // Commit implements core.ChainIndexerBackend @@ -181,16 +221,15 @@ func (c *ChtIndexerBackend) Commit() error { c.triedb.Commit(root, false) if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { - log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", c.lastHash, "root", root) + log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) } StoreChtRoot(c.diskdb, c.section, c.lastHash, root) return nil } const ( - BloomTrieFrequency = 32768 - ethBloomBitsSection = 4096 - ethBloomBitsConfirmations = 256 + BloomTrieFrequency = 32768 + ethBloomBitsSection = 4096 ) var ( @@ -215,7 +254,8 @@ func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root // BloomTrieIndexerBackend implements core.ChainIndexerBackend type BloomTrieIndexerBackend struct { - diskdb ethdb.Database + diskdb, trieTable ethdb.Database + odr OdrBackend triedb *trie.Database section, parentSectionSize, bloomTrieRatio uint64 trie *trie.Trie @@ -223,44 +263,98 @@ type BloomTrieIndexerBackend struct { } // NewBloomTrieIndexer creates a BloomTrie chain indexer -func NewBloomTrieIndexer(db ethdb.Database, clientMode bool) *core.ChainIndexer { +func NewBloomTrieIndexer(db ethdb.Database, clientMode bool, odr OdrBackend) *core.ChainIndexer { + trieTable := ethdb.NewTable(db, BloomTrieTablePrefix) backend := &BloomTrieIndexerBackend{ - diskdb: db, - triedb: trie.NewDatabase(ethdb.NewTable(db, BloomTrieTablePrefix)), + diskdb: db, + odr: odr, + trieTable: trieTable, + triedb: trie.NewDatabase(trieTable), } idb := ethdb.NewTable(db, "bltIndex-") - var confirmReq uint64 if clientMode { backend.parentSectionSize = BloomTrieFrequency - confirmReq = HelperTrieConfirmations } else { backend.parentSectionSize = ethBloomBitsSection - confirmReq = HelperTrieProcessConfirmations } backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) - return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, confirmReq-ethBloomBitsConfirmations, time.Millisecond*100, "bloomtrie") + return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, 0, time.Millisecond*100, "bloomtrie") +} + +// fetchMissingNodes tries to retrieve the last entries of the latest trusted bloom trie from the +// ODR backend in order to be able to add new entries and calculate subsequent root hashes +func (b *BloomTrieIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error { + indexCh := make(chan uint, types.BloomBitLength) + type res struct { + nodes *NodeSet + err error + } + resCh := make(chan res, types.BloomBitLength) + for i := 0; i < 20; i++ { + go func() { + for bitIndex := range indexCh { + r := &BloomRequest{BloomTrieRoot: root, BloomTrieNum: section - 1, BitIdx: bitIndex, SectionIdxList: []uint64{section - 1}} + for { + if err := b.odr.Retrieve(ctx, r); err == ErrNoPeers { + // if there are no peers to serve, retry later + select { + case <-ctx.Done(): + resCh <- res{nil, ctx.Err()} + return + case <-time.After(time.Second * 10): + // stay in the loop and try again + } + } else { + resCh <- res{r.Proofs, err} + break + } + } + } + }() + } + + for i := uint(0); i < types.BloomBitLength; i++ { + indexCh <- i + } + close(indexCh) + batch := b.trieTable.NewBatch() + for i := uint(0); i < types.BloomBitLength; i++ { + res := <-resCh + if res.err != nil { + return res.err + } + res.nodes.Store(batch) + } + return batch.Write() } // Reset implements core.ChainIndexerBackend -func (b *BloomTrieIndexerBackend) Reset(section uint64, lastSectionHead common.Hash) error { +func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { var root common.Hash if section > 0 { root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) } var err error b.trie, err = trie.New(root, b.triedb) + if err != nil && b.odr != nil { + err = b.fetchMissingNodes(ctx, section, root) + if err == nil { + b.trie, err = trie.New(root, b.triedb) + } + } b.section = section return err } // Process implements core.ChainIndexerBackend -func (b *BloomTrieIndexerBackend) Process(header *types.Header) { +func (b *BloomTrieIndexerBackend) Process(ctx context.Context, header *types.Header) error { num := header.Number.Uint64() - b.section*BloomTrieFrequency if (num+1)%b.parentSectionSize == 0 { b.sectionHeads[num/b.parentSectionSize] = header.Hash() } + return nil } // Commit implements core.ChainIndexerBackend @@ -300,7 +394,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { b.triedb.Commit(root, false) sectionHead := b.sectionHeads[b.bloomTrieRatio-1] - log.Info("Storing bloom trie", "section", b.section, "head", sectionHead, "root", root, "compression", float64(compSize)/float64(decompSize)) + log.Info("Storing bloom trie", "section", b.section, "head", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression", float64(compSize)/float64(decompSize)) StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) return nil From b24fb76a3ae8b2a41bde9ed0744b4284e385e011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 16 Aug 2018 09:41:16 +0300 Subject: [PATCH 15/96] cmd/puppeth: fix nil panic on disconnected stats gathering --- cmd/puppeth/wizard_netstats.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/puppeth/wizard_netstats.go b/cmd/puppeth/wizard_netstats.go index 89b38e262b..99ca11bb17 100644 --- a/cmd/puppeth/wizard_netstats.go +++ b/cmd/puppeth/wizard_netstats.go @@ -82,7 +82,6 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s logger.Info("Starting remote server health-check") stat := &serverStat{ - address: client.address, services: make(map[string]map[string]string), } if client == nil { @@ -94,6 +93,8 @@ func (w *wizard) gatherStats(server string, pubkey []byte, client *sshClient) *s } client = conn } + stat.address = client.address + // Client connected one way or another, run health-checks logger.Debug("Checking for nginx availability") if infos, err := checkNginx(client, w.network); err != nil { @@ -214,6 +215,9 @@ func (stats serverStats) render() { if len(stat.address) > len(separator[1]) { separator[1] = strings.Repeat("-", len(stat.address)) } + if len(stat.failure) > len(separator[1]) { + separator[1] = strings.Repeat("-", len(stat.failure)) + } for service, configs := range stat.services { if len(service) > len(separator[2]) { separator[2] = strings.Repeat("-", len(service)) @@ -250,7 +254,11 @@ func (stats serverStats) render() { sort.Strings(services) if len(services) == 0 { - table.Append([]string{server, stats[server].address, "", "", ""}) + if stats[server].failure != "" { + table.Append([]string{server, stats[server].failure, "", "", ""}) + } else { + table.Append([]string{server, stats[server].address, "", "", ""}) + } } for j, service := range services { // Add an empty line between all services From 3e21adc6488be41ac882c316486573374785cc82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 9 Aug 2018 13:46:52 +0300 Subject: [PATCH 16/96] crypto/bn256: fix issues caused by Go 1.11 --- crypto/bn256/cloudflare/gfp_amd64.s | 2 +- crypto/bn256/cloudflare/gfp_decl.go | 7 +++ crypto/bn256/google/bn256.go | 40 ++++++++++------ crypto/bn256/google/curve.go | 10 +++- crypto/bn256/google/twist.go | 10 +++- vendor/golang.org/x/sys/cpu/cpu.go | 38 +++++++++++++++ vendor/golang.org/x/sys/cpu/cpu_arm.go | 7 +++ vendor/golang.org/x/sys/cpu/cpu_arm64.go | 7 +++ vendor/golang.org/x/sys/cpu/cpu_gc_x86.go | 16 +++++++ vendor/golang.org/x/sys/cpu/cpu_gccgo.c | 43 +++++++++++++++++ vendor/golang.org/x/sys/cpu/cpu_gccgo.go | 26 ++++++++++ vendor/golang.org/x/sys/cpu/cpu_mips64x.go | 9 ++++ vendor/golang.org/x/sys/cpu/cpu_mipsx.go | 9 ++++ vendor/golang.org/x/sys/cpu/cpu_ppc64x.go | 9 ++++ vendor/golang.org/x/sys/cpu/cpu_s390x.go | 7 +++ vendor/golang.org/x/sys/cpu/cpu_x86.go | 55 ++++++++++++++++++++++ vendor/golang.org/x/sys/cpu/cpu_x86.s | 27 +++++++++++ vendor/vendor.json | 6 +++ 18 files changed, 311 insertions(+), 17 deletions(-) create mode 100644 vendor/golang.org/x/sys/cpu/cpu.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_arm.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_arm64.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_gc_x86.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_gccgo.c create mode 100644 vendor/golang.org/x/sys/cpu/cpu_gccgo.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_mips64x.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_mipsx.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_ppc64x.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_s390x.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_x86.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_x86.s diff --git a/crypto/bn256/cloudflare/gfp_amd64.s b/crypto/bn256/cloudflare/gfp_amd64.s index 3a785d200b..bdb4ffb787 100644 --- a/crypto/bn256/cloudflare/gfp_amd64.s +++ b/crypto/bn256/cloudflare/gfp_amd64.s @@ -110,7 +110,7 @@ TEXT ·gfpMul(SB),0,$160-24 MOVQ b+16(FP), SI // Jump to a slightly different implementation if MULX isn't supported. - CMPB runtime·support_bmi2(SB), $0 + CMPB ·hasBMI2(SB), $0 JE nobmi2Mul mulBMI2(0(DI),8(DI),16(DI),24(DI), 0(SI)) diff --git a/crypto/bn256/cloudflare/gfp_decl.go b/crypto/bn256/cloudflare/gfp_decl.go index 6a8a4fddb6..fdea5c11a5 100644 --- a/crypto/bn256/cloudflare/gfp_decl.go +++ b/crypto/bn256/cloudflare/gfp_decl.go @@ -5,6 +5,13 @@ package bn256 // This file contains forward declarations for the architecture-specific // assembly implementations of these functions, provided that they exist. +import ( + "golang.org/x/sys/cpu" +) + +//nolint:varcheck +var hasBMI2 = cpu.X86.HasBMI2 + // go:noescape func gfpNeg(c, a *gfP) diff --git a/crypto/bn256/google/bn256.go b/crypto/bn256/google/bn256.go index 5da83e033a..e0402e51f0 100644 --- a/crypto/bn256/google/bn256.go +++ b/crypto/bn256/google/bn256.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package bn256 implements a particular bilinear group at the 128-bit security level. +// Package bn256 implements a particular bilinear group. // // Bilinear groups are the basis of many of the new cryptographic protocols // that have been proposed over the past decade. They consist of a triplet of @@ -14,6 +14,10 @@ // Barreto-Naehrig curve as described in // http://cryptojedi.org/papers/dclxvi-20100714.pdf. Its output is compatible // with the implementation described in that paper. +// +// (This package previously claimed to operate at a 128-bit security level. +// However, recent improvements in attacks mean that is no longer true. See +// https://moderncrypto.org/mail-archive/curves/2016/000740.html.) package bn256 import ( @@ -50,8 +54,8 @@ func RandomG1(r io.Reader) (*big.Int, *G1, error) { return k, new(G1).ScalarBaseMult(k), nil } -func (g *G1) String() string { - return "bn256.G1" + g.p.String() +func (e *G1) String() string { + return "bn256.G1" + e.p.String() } // CurvePoints returns p's curve points in big integer @@ -98,15 +102,19 @@ func (e *G1) Neg(a *G1) *G1 { } // Marshal converts n to a byte slice. -func (n *G1) Marshal() []byte { - n.p.MakeAffine(nil) - - xBytes := new(big.Int).Mod(n.p.x, P).Bytes() - yBytes := new(big.Int).Mod(n.p.y, P).Bytes() - +func (e *G1) Marshal() []byte { // Each value is a 256-bit number. const numBytes = 256 / 8 + if e.p.IsInfinity() { + return make([]byte, numBytes*2) + } + + e.p.MakeAffine(nil) + + xBytes := new(big.Int).Mod(e.p.x, P).Bytes() + yBytes := new(big.Int).Mod(e.p.y, P).Bytes() + ret := make([]byte, numBytes*2) copy(ret[1*numBytes-len(xBytes):], xBytes) copy(ret[2*numBytes-len(yBytes):], yBytes) @@ -175,8 +183,8 @@ func RandomG2(r io.Reader) (*big.Int, *G2, error) { return k, new(G2).ScalarBaseMult(k), nil } -func (g *G2) String() string { - return "bn256.G2" + g.p.String() +func (e *G2) String() string { + return "bn256.G2" + e.p.String() } // CurvePoints returns the curve points of p which includes the real @@ -216,6 +224,13 @@ func (e *G2) Add(a, b *G2) *G2 { // Marshal converts n into a byte slice. func (n *G2) Marshal() []byte { + // Each value is a 256-bit number. + const numBytes = 256 / 8 + + if n.p.IsInfinity() { + return make([]byte, numBytes*4) + } + n.p.MakeAffine(nil) xxBytes := new(big.Int).Mod(n.p.x.x, P).Bytes() @@ -223,9 +238,6 @@ func (n *G2) Marshal() []byte { yxBytes := new(big.Int).Mod(n.p.y.x, P).Bytes() yyBytes := new(big.Int).Mod(n.p.y.y, P).Bytes() - // Each value is a 256-bit number. - const numBytes = 256 / 8 - ret := make([]byte, numBytes*4) copy(ret[1*numBytes-len(xxBytes):], xxBytes) copy(ret[2*numBytes-len(xyBytes):], xyBytes) diff --git a/crypto/bn256/google/curve.go b/crypto/bn256/google/curve.go index 3e679fdc7e..819cb81da7 100644 --- a/crypto/bn256/google/curve.go +++ b/crypto/bn256/google/curve.go @@ -245,11 +245,19 @@ func (c *curvePoint) Mul(a *curvePoint, scalar *big.Int, pool *bnPool) *curvePoi return c } +// MakeAffine converts c to affine form and returns c. If c is ∞, then it sets +// c to 0 : 1 : 0. func (c *curvePoint) MakeAffine(pool *bnPool) *curvePoint { if words := c.z.Bits(); len(words) == 1 && words[0] == 1 { return c } - + if c.IsInfinity() { + c.x.SetInt64(0) + c.y.SetInt64(1) + c.z.SetInt64(0) + c.t.SetInt64(0) + return c + } zInv := pool.Get().ModInverse(c.z, P) t := pool.Get().Mul(c.y, zInv) t.Mod(t, P) diff --git a/crypto/bn256/google/twist.go b/crypto/bn256/google/twist.go index 1f5a4d9deb..43364ff5b7 100644 --- a/crypto/bn256/google/twist.go +++ b/crypto/bn256/google/twist.go @@ -225,11 +225,19 @@ func (c *twistPoint) Mul(a *twistPoint, scalar *big.Int, pool *bnPool) *twistPoi return c } +// MakeAffine converts c to affine form and returns c. If c is ∞, then it sets +// c to 0 : 1 : 0. func (c *twistPoint) MakeAffine(pool *bnPool) *twistPoint { if c.z.IsOne() { return c } - + if c.IsInfinity() { + c.x.SetZero() + c.y.SetOne() + c.z.SetZero() + c.t.SetZero() + return c + } zInv := newGFp2(pool).Invert(c.z, pool) t := newGFp2(pool).Mul(c.y, zInv, pool) zInv2 := newGFp2(pool).Square(zInv, pool) diff --git a/vendor/golang.org/x/sys/cpu/cpu.go b/vendor/golang.org/x/sys/cpu/cpu.go new file mode 100644 index 0000000000..3d88f86673 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu.go @@ -0,0 +1,38 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cpu implements processor feature detection for +// various CPU architectures. +package cpu + +// CacheLinePad is used to pad structs to avoid false sharing. +type CacheLinePad struct{ _ [cacheLineSize]byte } + +// X86 contains the supported CPU features of the +// current X86/AMD64 platform. If the current platform +// is not X86/AMD64 then all feature flags are false. +// +// X86 is padded to avoid false sharing. Further the HasAVX +// and HasAVX2 are only set if the OS supports XMM and YMM +// registers in addition to the CPUID feature bit being set. +var X86 struct { + _ CacheLinePad + HasAES bool // AES hardware implementation (AES NI) + HasADX bool // Multi-precision add-carry instruction extensions + HasAVX bool // Advanced vector extension + HasAVX2 bool // Advanced vector extension 2 + HasBMI1 bool // Bit manipulation instruction set 1 + HasBMI2 bool // Bit manipulation instruction set 2 + HasERMS bool // Enhanced REP for MOVSB and STOSB + HasFMA bool // Fused-multiply-add instructions + HasOSXSAVE bool // OS supports XSAVE/XRESTOR for saving/restoring XMM registers. + HasPCLMULQDQ bool // PCLMULQDQ instruction - most often used for AES-GCM + HasPOPCNT bool // Hamming weight instruction POPCNT. + HasSSE2 bool // Streaming SIMD extension 2 (always available on amd64) + HasSSE3 bool // Streaming SIMD extension 3 + HasSSSE3 bool // Supplemental streaming SIMD extension 3 + HasSSE41 bool // Streaming SIMD extension 4 and 4.1 + HasSSE42 bool // Streaming SIMD extension 4 and 4.2 + _ CacheLinePad +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm.go b/vendor/golang.org/x/sys/cpu/cpu_arm.go new file mode 100644 index 0000000000..d93036f752 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_arm.go @@ -0,0 +1,7 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 32 diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_arm64.go new file mode 100644 index 0000000000..1d2ab2902a --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_arm64.go @@ -0,0 +1,7 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 64 diff --git a/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go new file mode 100644 index 0000000000..f7cb46971c --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go @@ -0,0 +1,16 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 +// +build !gccgo + +package cpu + +// cpuid is implemented in cpu_x86.s for gc compiler +// and in cpu_gccgo.c for gccgo. +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) + +// xgetbv with ecx = 0 is implemented in cpu_x86.s for gc compiler +// and in cpu_gccgo.c for gccgo. +func xgetbv() (eax, edx uint32) diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo.c b/vendor/golang.org/x/sys/cpu/cpu_gccgo.c new file mode 100644 index 0000000000..e363c7d131 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo.c @@ -0,0 +1,43 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 +// +build gccgo + +#include +#include + +// Need to wrap __get_cpuid_count because it's declared as static. +int +gccgoGetCpuidCount(uint32_t leaf, uint32_t subleaf, + uint32_t *eax, uint32_t *ebx, + uint32_t *ecx, uint32_t *edx) +{ + return __get_cpuid_count(leaf, subleaf, eax, ebx, ecx, edx); +} + +// xgetbv reads the contents of an XCR (Extended Control Register) +// specified in the ECX register into registers EDX:EAX. +// Currently, the only supported value for XCR is 0. +// +// TODO: Replace with a better alternative: +// +// #include +// +// #pragma GCC target("xsave") +// +// void gccgoXgetbv(uint32_t *eax, uint32_t *edx) { +// unsigned long long x = _xgetbv(0); +// *eax = x & 0xffffffff; +// *edx = (x >> 32) & 0xffffffff; +// } +// +// Note that _xgetbv is defined starting with GCC 8. +void +gccgoXgetbv(uint32_t *eax, uint32_t *edx) +{ + __asm(" xorl %%ecx, %%ecx\n" + " xgetbv" + : "=a"(*eax), "=d"(*edx)); +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo.go b/vendor/golang.org/x/sys/cpu/cpu_gccgo.go new file mode 100644 index 0000000000..ba49b91bd3 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo.go @@ -0,0 +1,26 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 +// +build gccgo + +package cpu + +//extern gccgoGetCpuidCount +func gccgoGetCpuidCount(eaxArg, ecxArg uint32, eax, ebx, ecx, edx *uint32) + +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) { + var a, b, c, d uint32 + gccgoGetCpuidCount(eaxArg, ecxArg, &a, &b, &c, &d) + return a, b, c, d +} + +//extern gccgoXgetbv +func gccgoXgetbv(eax, edx *uint32) + +func xgetbv() (eax, edx uint32) { + var a, d uint32 + gccgoXgetbv(&a, &d) + return a, d +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_mips64x.go b/vendor/golang.org/x/sys/cpu/cpu_mips64x.go new file mode 100644 index 0000000000..6165f12124 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_mips64x.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips64 mips64le + +package cpu + +const cacheLineSize = 32 diff --git a/vendor/golang.org/x/sys/cpu/cpu_mipsx.go b/vendor/golang.org/x/sys/cpu/cpu_mipsx.go new file mode 100644 index 0000000000..1269eee88d --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_mipsx.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips mipsle + +package cpu + +const cacheLineSize = 32 diff --git a/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go b/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go new file mode 100644 index 0000000000..d10759a524 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le + +package cpu + +const cacheLineSize = 128 diff --git a/vendor/golang.org/x/sys/cpu/cpu_s390x.go b/vendor/golang.org/x/sys/cpu/cpu_s390x.go new file mode 100644 index 0000000000..684c4f005d --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_s390x.go @@ -0,0 +1,7 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 256 diff --git a/vendor/golang.org/x/sys/cpu/cpu_x86.go b/vendor/golang.org/x/sys/cpu/cpu_x86.go new file mode 100644 index 0000000000..71e288b062 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_x86.go @@ -0,0 +1,55 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 + +package cpu + +const cacheLineSize = 64 + +func init() { + maxID, _, _, _ := cpuid(0, 0) + + if maxID < 1 { + return + } + + _, _, ecx1, edx1 := cpuid(1, 0) + X86.HasSSE2 = isSet(26, edx1) + + X86.HasSSE3 = isSet(0, ecx1) + X86.HasPCLMULQDQ = isSet(1, ecx1) + X86.HasSSSE3 = isSet(9, ecx1) + X86.HasFMA = isSet(12, ecx1) + X86.HasSSE41 = isSet(19, ecx1) + X86.HasSSE42 = isSet(20, ecx1) + X86.HasPOPCNT = isSet(23, ecx1) + X86.HasAES = isSet(25, ecx1) + X86.HasOSXSAVE = isSet(27, ecx1) + + osSupportsAVX := false + // For XGETBV, OSXSAVE bit is required and sufficient. + if X86.HasOSXSAVE { + eax, _ := xgetbv() + // Check if XMM and YMM registers have OS support. + osSupportsAVX = isSet(1, eax) && isSet(2, eax) + } + + X86.HasAVX = isSet(28, ecx1) && osSupportsAVX + + if maxID < 7 { + return + } + + _, ebx7, _, _ := cpuid(7, 0) + X86.HasBMI1 = isSet(3, ebx7) + X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX + X86.HasBMI2 = isSet(8, ebx7) + X86.HasERMS = isSet(9, ebx7) + X86.HasADX = isSet(19, ebx7) +} + +func isSet(bitpos uint, value uint32) bool { + return value&(1< Date: Thu, 16 Aug 2018 19:14:33 +0800 Subject: [PATCH 17/96] miner: regenerate mining work every 3 seconds (#17413) * miner: regenerate mining work every 3 seconds * miner: polish --- core/events.go | 3 - miner/worker.go | 332 ++++++++++++++++++++++++++----------------- miner/worker_test.go | 65 +++++++++ 3 files changed, 267 insertions(+), 133 deletions(-) diff --git a/core/events.go b/core/events.go index 8d200f2a29..710bdb5894 100644 --- a/core/events.go +++ b/core/events.go @@ -29,9 +29,6 @@ type PendingLogsEvent struct { Logs []*types.Log } -// PendingStateEvent is posted pre mining and notifies of pending state changes. -type PendingStateEvent struct{} - // NewMinedBlockEvent is posted when a block has been imported. type NewMinedBlockEvent struct{ Block *types.Block } diff --git a/miner/worker.go b/miner/worker.go index fae480c84c..81a9eabfd5 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -40,19 +40,27 @@ import ( const ( // resultQueueSize is the size of channel listening to sealing result. resultQueueSize = 10 + // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 + // chainHeadChanSize is the size of channel listening to ChainHeadEvent. chainHeadChanSize = 10 + // chainSideChanSize is the size of channel listening to ChainSideEvent. chainSideChanSize = 10 - miningLogAtDepth = 5 + + // miningLogAtDepth is the number of confirmations before logging successful mining. + miningLogAtDepth = 5 + + // blockRecommitInterval is the time interval to recreate the mining block with + // any newly arrived transactions. + blockRecommitInterval = 3 * time.Second ) -// Env is the worker's current environment and holds all of the current state information. -type Env struct { - config *params.ChainConfig +// environment is the worker's current environment and holds all of the current state information. +type environment struct { signer types.Signer state *state.StateDB // apply state changes here @@ -67,105 +75,6 @@ type Env struct { receipts []*types.Receipt } -func (env *Env) commitTransaction(tx *types.Transaction, bc *core.BlockChain, coinbase common.Address, gp *core.GasPool) (error, []*types.Log) { - snap := env.state.Snapshot() - - receipt, _, err := core.ApplyTransaction(env.config, bc, &coinbase, gp, env.state, env.header, tx, &env.header.GasUsed, vm.Config{}) - if err != nil { - env.state.RevertToSnapshot(snap) - return err, nil - } - env.txs = append(env.txs, tx) - env.receipts = append(env.receipts, receipt) - - return nil, receipt.Logs -} - -func (env *Env) commitTransactions(mux *event.TypeMux, txs *types.TransactionsByPriceAndNonce, bc *core.BlockChain, coinbase common.Address) { - if env.gasPool == nil { - env.gasPool = new(core.GasPool).AddGas(env.header.GasLimit) - } - - var coalescedLogs []*types.Log - - for { - // If we don't have enough gas for any further transactions then we're done - if env.gasPool.Gas() < params.TxGas { - log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) - break - } - // Retrieve the next transaction and abort if all done - tx := txs.Peek() - if tx == nil { - break - } - // Error may be ignored here. The error has already been checked - // during transaction acceptance is the transaction pool. - // - // We use the eip155 signer regardless of the current hf. - from, _ := types.Sender(env.signer, tx) - // Check whether the tx is replay protected. If we're not in the EIP155 hf - // phase, start ignoring the sender until we do. - if tx.Protected() && !env.config.IsEIP155(env.header.Number) { - log.Trace("Ignoring reply protected transaction", "hash", tx.Hash(), "eip155", env.config.EIP155Block) - - txs.Pop() - continue - } - // Start executing the transaction - env.state.Prepare(tx.Hash(), common.Hash{}, env.tcount) - - err, logs := env.commitTransaction(tx, bc, coinbase, env.gasPool) - switch err { - case core.ErrGasLimitReached: - // Pop the current out-of-gas transaction without shifting in the next from the account - log.Trace("Gas limit exceeded for current block", "sender", from) - txs.Pop() - - case core.ErrNonceTooLow: - // New head notification data race between the transaction pool and miner, shift - log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce()) - txs.Shift() - - case core.ErrNonceTooHigh: - // Reorg notification data race between the transaction pool and miner, skip account = - log.Trace("Skipping account with hight nonce", "sender", from, "nonce", tx.Nonce()) - txs.Pop() - - case nil: - // Everything ok, collect the logs and shift in the next transaction from the same account - coalescedLogs = append(coalescedLogs, logs...) - env.tcount++ - txs.Shift() - - default: - // Strange error, discard the transaction and get the next in line (note, the - // nonce-too-high clause will prevent us from executing in vain). - log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err) - txs.Shift() - } - } - - if len(coalescedLogs) > 0 || env.tcount > 0 { - // make a copy, the state caches the logs and these logs get "upgraded" from pending to mined - // logs by filling in the block hash when the block was mined by the local miner. This can - // cause a race condition if a log was "upgraded" before the PendingLogsEvent is processed. - cpy := make([]*types.Log, len(coalescedLogs)) - for i, l := range coalescedLogs { - cpy[i] = new(types.Log) - *cpy[i] = *l - } - go func(logs []*types.Log, tcount int) { - if len(logs) > 0 { - mux.Post(core.PendingLogsEvent{Logs: logs}) - } - if tcount > 0 { - mux.Post(core.PendingStateEvent{}) - } - }(cpy, env.tcount) - } -} - // task contains all information for consensus engine sealing and result submitting. type task struct { receipts []*types.Receipt @@ -174,6 +83,17 @@ type task struct { createdAt time.Time } +const ( + commitInterruptNone int32 = iota + commitInterruptNewHead + commitInterruptResubmit +) + +type newWorkReq struct { + interrupt *int32 + noempty bool +} + // worker is the main object which takes care of submitting new work to consensus engine // and gathering the sealing result. type worker struct { @@ -192,12 +112,13 @@ type worker struct { chainSideSub event.Subscription // Channels - newWork chan struct{} - taskCh chan *task - resultCh chan *task - exitCh chan struct{} + newWorkCh chan *newWorkReq + taskCh chan *task + resultCh chan *task + startCh chan struct{} + exitCh chan struct{} - current *Env // An environment for current running cycle. + current *environment // An environment for current running cycle. possibleUncles map[common.Hash]*types.Block // A set of side blocks as the possible uncle blocks. unconfirmed *unconfirmedBlocks // A set of locally mined blocks pending canonicalness confirmations. @@ -230,10 +151,11 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, txsCh: make(chan core.NewTxsEvent, txChanSize), chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), - newWork: make(chan struct{}, 1), + newWorkCh: make(chan *newWorkReq), taskCh: make(chan *task), resultCh: make(chan *task, resultQueueSize), exitCh: make(chan struct{}), + startCh: make(chan struct{}, 1), } // Subscribe NewTxsEvent for tx pool worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) @@ -242,11 +164,13 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) go worker.mainLoop() + go worker.newWorkLoop() go worker.resultLoop() go worker.taskLoop() // Submit first work to initialize pending state. - worker.newWork <- struct{}{} + worker.startCh <- struct{}{} + return worker } @@ -286,7 +210,7 @@ func (w *worker) pendingBlock() *types.Block { // start sets the running status as 1 and triggers new work submitting. func (w *worker) start() { atomic.StoreInt32(&w.running, 1) - w.newWork <- struct{}{} + w.startCh <- struct{}{} } // stop sets the running status as 0. @@ -313,6 +237,44 @@ func (w *worker) close() { } } +// newWorkLoop is a standalone goroutine to submit new mining work upon received events. +func (w *worker) newWorkLoop() { + var interrupt *int32 + + timer := time.NewTimer(0) + <-timer.C // discard the initial tick + + // recommit aborts in-flight transaction execution with given signal and resubmits a new one. + recommit := func(noempty bool, s int32) { + if interrupt != nil { + atomic.StoreInt32(interrupt, s) + } + interrupt = new(int32) + w.newWorkCh <- &newWorkReq{interrupt: interrupt, noempty: noempty} + timer.Reset(blockRecommitInterval) + } + + for { + select { + case <-w.startCh: + recommit(false, commitInterruptNewHead) + + case <-w.chainHeadCh: + recommit(false, commitInterruptNewHead) + + case <-timer.C: + // If mining is running resubmit a new work cycle periodically to pull in + // higher priced transactions. Disable this overhead for pending blocks. + if w.isRunning() && (w.config.Clique == nil || w.config.Clique.Period > 0) { + recommit(true, commitInterruptResubmit) + } + + case <-w.exitCh: + return + } + } +} + // mainLoop is a standalone goroutine to regenerate the sealing task based on the received event. func (w *worker) mainLoop() { defer w.txsSub.Unsubscribe() @@ -321,13 +283,8 @@ func (w *worker) mainLoop() { for { select { - case <-w.newWork: - // Submit a work when the worker is created or started. - w.commitNewWork() - - case <-w.chainHeadCh: - // Resubmit a work for new cycle once worker receives chain head event. - w.commitNewWork() + case req := <-w.newWorkCh: + w.commitNewWork(req.interrupt, req.noempty) case ev := <-w.chainSideCh: if _, exist := w.possibleUncles[ev.Block.Hash()]; exist { @@ -364,9 +321,9 @@ func (w *worker) mainLoop() { // already included in the current mining block. These transactions will // be automatically eliminated. if !w.isRunning() && w.current != nil { - w.mu.Lock() + w.mu.RLock() coinbase := w.coinbase - w.mu.Unlock() + w.mu.RUnlock() txs := make(map[common.Address]types.Transactions) for _, tx := range ev.Txs { @@ -374,12 +331,12 @@ func (w *worker) mainLoop() { txs[acc] = append(txs[acc], tx) } txset := types.NewTransactionsByPriceAndNonce(w.current.signer, txs) - w.current.commitTransactions(w.mux, txset, w.chain, coinbase) + w.commitTransactions(txset, coinbase, nil) w.updateSnapshot() } else { // If we're mining, but nothing is being processed, wake on new transactions if w.config.Clique != nil && w.config.Clique.Period == 0 { - w.commitNewWork() + w.commitNewWork(nil, false) } } @@ -508,8 +465,7 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { if err != nil { return err } - env := &Env{ - config: w.config, + env := &environment{ signer: types.NewEIP155Signer(w.config.ChainID), state: state, ancestors: mapset.NewSet(), @@ -534,7 +490,7 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { } // commitUncle adds the given block to uncle block set, returns error if failed to add. -func (w *worker) commitUncle(env *Env, uncle *types.Header) error { +func (w *worker) commitUncle(env *environment, uncle *types.Header) error { hash := uncle.Hash() if env.uncles.Contains(hash) { return fmt.Errorf("uncle not unique") @@ -579,8 +535,120 @@ func (w *worker) updateSnapshot() { w.snapshotState = w.current.state.Copy() } +func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address) ([]*types.Log, error) { + snap := w.current.state.Snapshot() + + receipt, _, err := core.ApplyTransaction(w.config, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, vm.Config{}) + if err != nil { + w.current.state.RevertToSnapshot(snap) + return nil, err + } + w.current.txs = append(w.current.txs, tx) + w.current.receipts = append(w.current.receipts, receipt) + + return receipt.Logs, nil +} + +func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coinbase common.Address, interrupt *int32) bool { + // Short circuit if current is nil + if w.current == nil { + return true + } + + if w.current.gasPool == nil { + w.current.gasPool = new(core.GasPool).AddGas(w.current.header.GasLimit) + } + + var coalescedLogs []*types.Log + + for { + // In the following three cases, we will interrupt the execution of the transaction. + // (1) new head block event arrival, the interrupt signal is 1 + // (2) worker start or restart, the interrupt signal is 1 + // (3) worker recreate the mining block with any newly arrived transactions, the interrupt signal is 2. + // For the first two cases, the semi-finished work will be discarded. + // For the third case, the semi-finished work will be submitted to the consensus engine. + // TODO(rjl493456442) give feedback to newWorkLoop to adjust resubmit interval if it is too short. + if interrupt != nil && atomic.LoadInt32(interrupt) != commitInterruptNone { + return atomic.LoadInt32(interrupt) == commitInterruptNewHead + } + // If we don't have enough gas for any further transactions then we're done + if w.current.gasPool.Gas() < params.TxGas { + log.Trace("Not enough gas for further transactions", "have", w.current.gasPool, "want", params.TxGas) + break + } + // Retrieve the next transaction and abort if all done + tx := txs.Peek() + if tx == nil { + break + } + // Error may be ignored here. The error has already been checked + // during transaction acceptance is the transaction pool. + // + // We use the eip155 signer regardless of the current hf. + from, _ := types.Sender(w.current.signer, tx) + // Check whether the tx is replay protected. If we're not in the EIP155 hf + // phase, start ignoring the sender until we do. + if tx.Protected() && !w.config.IsEIP155(w.current.header.Number) { + log.Trace("Ignoring reply protected transaction", "hash", tx.Hash(), "eip155", w.config.EIP155Block) + + txs.Pop() + continue + } + // Start executing the transaction + w.current.state.Prepare(tx.Hash(), common.Hash{}, w.current.tcount) + + logs, err := w.commitTransaction(tx, coinbase) + switch err { + case core.ErrGasLimitReached: + // Pop the current out-of-gas transaction without shifting in the next from the account + log.Trace("Gas limit exceeded for current block", "sender", from) + txs.Pop() + + case core.ErrNonceTooLow: + // New head notification data race between the transaction pool and miner, shift + log.Trace("Skipping transaction with low nonce", "sender", from, "nonce", tx.Nonce()) + txs.Shift() + + case core.ErrNonceTooHigh: + // Reorg notification data race between the transaction pool and miner, skip account = + log.Trace("Skipping account with hight nonce", "sender", from, "nonce", tx.Nonce()) + txs.Pop() + + case nil: + // Everything ok, collect the logs and shift in the next transaction from the same account + coalescedLogs = append(coalescedLogs, logs...) + w.current.tcount++ + txs.Shift() + + default: + // Strange error, discard the transaction and get the next in line (note, the + // nonce-too-high clause will prevent us from executing in vain). + log.Debug("Transaction failed, account skipped", "hash", tx.Hash(), "err", err) + txs.Shift() + } + } + + if !w.isRunning() && len(coalescedLogs) > 0 { + // We don't push the pendingLogsEvent while we are mining. The reason is that + // when we are mining, the worker will regenerate a mining block every 3 seconds. + // In order to avoid pushing the repeated pendingLog, we disable the pending log pushing. + + // make a copy, the state caches the logs and these logs get "upgraded" from pending to mined + // logs by filling in the block hash when the block was mined by the local miner. This can + // cause a race condition if a log was "upgraded" before the PendingLogsEvent is processed. + cpy := make([]*types.Log, len(coalescedLogs)) + for i, l := range coalescedLogs { + cpy[i] = new(types.Log) + *cpy[i] = *l + } + go w.mux.Post(core.PendingLogsEvent{Logs: cpy}) + } + return false +} + // commitNewWork generates several new sealing tasks based on the parent block. -func (w *worker) commitNewWork() { +func (w *worker) commitNewWork(interrupt *int32, noempty bool) { w.mu.RLock() defer w.mu.RUnlock() @@ -666,9 +734,11 @@ func (w *worker) commitNewWork() { delete(w.possibleUncles, hash) } - // Create an empty block based on temporary copied state for sealing in advance without waiting block - // execution finished. - w.commit(uncles, nil, false, tstart) + if !noempty { + // Create an empty block based on temporary copied state for sealing in advance without waiting block + // execution finished. + w.commit(uncles, nil, false, tstart) + } // Fill the block with all available pending transactions. pending, err := w.eth.TxPool().Pending() @@ -682,7 +752,9 @@ func (w *worker) commitNewWork() { return } txs := types.NewTransactionsByPriceAndNonce(w.current.signer, pending) - env.commitTransactions(w.mux, txs, w.chain, w.coinbase) + if w.commitTransactions(txs, w.coinbase, interrupt) { + return + } w.commit(uncles, w.fullTaskHook, true, tstart) } diff --git a/miner/worker_test.go b/miner/worker_test.go index 408c47e3b3..34bb7f5f33 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -270,3 +270,68 @@ func TestStreamUncleBlock(t *testing.T) { t.Error("new task timeout") } } + +func TestRegenerateMiningBlockEthash(t *testing.T) { + testRegenerateMiningBlock(t, ethashChainConfig, ethash.NewFaker()) +} + +func TestRegenerateMiningBlockClique(t *testing.T) { + testRegenerateMiningBlock(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) +} + +func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { + defer engine.Close() + + w, b := newTestWorker(t, chainConfig, engine) + defer w.close() + + var taskCh = make(chan struct{}) + + taskIndex := 0 + w.newTaskHook = func(task *task) { + if task.block.NumberU64() == 1 { + if taskIndex == 2 { + receiptLen, balance := 2, big.NewInt(2000) + if len(task.receipts) != receiptLen { + t.Errorf("receipt number mismatch has %d, want %d", len(task.receipts), receiptLen) + } + if task.state.GetBalance(acc1Addr).Cmp(balance) != 0 { + t.Errorf("account balance mismatch has %d, want %d", task.state.GetBalance(acc1Addr), balance) + } + } + taskCh <- struct{}{} + taskIndex += 1 + } + } + w.skipSealHook = func(task *task) bool { + return true + } + w.fullTaskHook = func() { + time.Sleep(100 * time.Millisecond) + } + // Ensure worker has finished initialization + for { + b := w.pendingBlock() + if b != nil && b.NumberU64() == 1 { + break + } + } + + w.start() + // Ignore the first two works + for i := 0; i < 2; i += 1 { + select { + case <-taskCh: + case <-time.NewTimer(time.Second).C: + t.Error("new task timeout") + } + } + b.txPool.AddLocals(newTxs) + time.Sleep(3 * time.Second) + + select { + case <-taskCh: + case <-time.NewTimer(time.Second).C: + t.Error("new task timeout") + } +} From 62f5137a72c69ae8cffbc3526611af77448d25de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 16 Aug 2018 14:47:49 +0300 Subject: [PATCH 18/96] miner: add gas and fee details to mining logs --- miner/worker.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 81a9eabfd5..e7e2796458 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -780,8 +780,16 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st select { case w.taskCh <- &task{receipts: receipts, state: s, block: block, createdAt: time.Now()}: w.unconfirmed.Shift(block.NumberU64() - 1) - log.Info("Commit new mining work", "number", block.Number(), "txs", w.current.tcount, "uncles", len(uncles), - "elapsed", common.PrettyDuration(time.Since(start))) + + feesWei := new(big.Int) + for _, tx := range block.Transactions() { + feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(tx.Gas()), tx.GasPrice())) + } + feesEth := new(big.Float).Quo(new(big.Float).SetInt(feesWei), new(big.Float).SetInt(big.NewInt(params.Ether))) + + log.Info("Commit new mining work", "number", block.Number(), "uncles", len(uncles), "txs", w.current.tcount, + "gas", block.GasUsed(), "fees", feesEth, "elapsed", common.PrettyDuration(time.Since(start))) + case <-w.exitCh: log.Info("Worker has exited") } From 60390878a5128e64228de6873b5a16adddbbd68b Mon Sep 17 00:00:00 2001 From: Sasuke1964 Date: Fri, 17 Aug 2018 02:38:02 -0500 Subject: [PATCH 19/96] accounts: fixed typo (#17421) --- accounts/accounts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/accounts/accounts.go b/accounts/accounts.go index 76951e1a42..cb1eae2815 100644 --- a/accounts/accounts.go +++ b/accounts/accounts.go @@ -106,7 +106,7 @@ type Wallet interface { // or optionally with the aid of any location metadata from the embedded URL field. // // If the wallet requires additional authentication to sign the request (e.g. - // a password to decrypt the account, or a PIN code o verify the transaction), + // a password to decrypt the account, or a PIN code to verify the transaction), // an AuthNeededError instance will be returned, containing infos for the user // about which fields or actions are needed. The user may retry by providing // the needed details via SignTxWithPassphrase, or by other means (e.g. unlock From f44046a1c6889049dbf0f9448075a43f5b280b09 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Fri, 17 Aug 2018 10:33:39 +0200 Subject: [PATCH 20/96] build: do not require `ethereum-swarm` deb when installing `ethereum` (#17425) --- build/ci.go | 11 ----------- build/deb/ethereum/deb.control | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/build/ci.go b/build/ci.go index ff23e15fdb..40252cbde1 100644 --- a/build/ci.go +++ b/build/ci.go @@ -644,17 +644,6 @@ func (meta debMetadata) ExeName(exe debExecutable) string { return exe.Package() } -// EthereumSwarmPackageName returns the name of the swarm package based on -// environment, e.g. "ethereum-swarm-unstable", or "ethereum-swarm". -// This is needed so that we make sure that "ethereum" package, -// depends on and installs "ethereum-swarm" -func (meta debMetadata) EthereumSwarmPackageName() string { - if isUnstableBuild(meta.Env) { - return debSwarm.Name + "-unstable" - } - return debSwarm.Name -} - // ExeConflicts returns the content of the Conflicts field // for executable packages. func (meta debMetadata) ExeConflicts(exe debExecutable) string { diff --git a/build/deb/ethereum/deb.control b/build/deb/ethereum/deb.control index e693d1d046..defb106fe3 100644 --- a/build/deb/ethereum/deb.control +++ b/build/deb/ethereum/deb.control @@ -10,7 +10,7 @@ Vcs-Browser: https://github.com/ethereum/go-ethereum Package: {{.Name}} Architecture: any -Depends: ${misc:Depends}, {{.EthereumSwarmPackageName}}, {{.ExeList}} +Depends: ${misc:Depends}, {{.ExeList}} Description: Meta-package to install geth, swarm, and other tools Meta-package to install geth, swarm and other tools From 2695fa2213fe5010a80970bca1078834662d5972 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 17 Aug 2018 12:21:53 +0200 Subject: [PATCH 21/96] les: fix crasher in NodeInfo when running as server (#17419) * les: fix crasher in NodeInfo when running as server The ProtocolManager computes CHT and Bloom trie roots by asking the indexers for their current head. It tried to get the indexers from LesOdr, but no LesOdr instance is created in server mode. Attempt to fix this by moving the indexers, protocol creation and NodeInfo to a new lesCommons struct which is embedded into both server and client. All this setup code should really be cleaned up, but this is just a hotfix so we have to do that some other time. * les: fix commons protocol maker --- les/backend.go | 48 ++++++++--------- les/commons.go | 106 +++++++++++++++++++++++++++++++++++++ les/handler.go | 128 ++++++++++----------------------------------- les/helper_test.go | 10 +--- les/server.go | 36 +++++++------ 5 files changed, 177 insertions(+), 151 deletions(-) create mode 100644 les/commons.go diff --git a/les/backend.go b/les/backend.go index 9b8cc1828f..d26c1470fe 100644 --- a/les/backend.go +++ b/les/backend.go @@ -34,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/light" @@ -47,26 +46,24 @@ import ( ) type LightEthereum struct { - config *eth.Config + lesCommons odr *LesOdr relay *LesTxRelay chainConfig *params.ChainConfig // Channel for shutting down the service shutdownChan chan bool - // Handlers - peers *peerSet - txPool *light.TxPool - blockchain *light.LightChain - protocolManager *ProtocolManager - serverPool *serverPool - reqDist *requestDistributor - retriever *retrieveManager - // DB interfaces - chainDb ethdb.Database // Block chain database - bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests - bloomIndexer, chtIndexer, bloomTrieIndexer *core.ChainIndexer + // Handlers + peers *peerSet + txPool *light.TxPool + blockchain *light.LightChain + serverPool *serverPool + reqDist *requestDistributor + retriever *retrieveManager + + bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests + bloomIndexer *core.ChainIndexer ApiBackend *LesApiBackend @@ -95,9 +92,11 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { quitSync := make(chan struct{}) leth := &LightEthereum{ - config: config, + lesCommons: lesCommons{ + chainDb: chainDb, + config: config, + }, chainConfig: chainConfig, - chainDb: chainDb, eventMux: ctx.EventMux, peers: peers, reqDist: newRequestDistributor(peers, quitSync), @@ -112,10 +111,12 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { leth.relay = NewLesTxRelay(peers, leth.reqDist) leth.serverPool = newServerPool(chainDb, quitSync, &leth.wg) leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool) + leth.odr = NewLesOdr(chainDb, leth.retriever) leth.chtIndexer = light.NewChtIndexer(chainDb, true, leth.odr) leth.bloomTrieIndexer = light.NewBloomTrieIndexer(chainDb, true, leth.odr) leth.odr.SetIndexers(leth.chtIndexer, leth.bloomTrieIndexer, leth.bloomIndexer) + // Note: NewLightChain adds the trusted checkpoint so it needs an ODR with // indexers already set but not started yet if leth.blockchain, err = light.NewLightChain(leth.odr, leth.chainConfig, leth.engine); err != nil { @@ -125,6 +126,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { leth.bloomIndexer.AddChildIndexer(leth.bloomTrieIndexer) leth.chtIndexer.Start(leth.blockchain) leth.bloomIndexer.Start(leth.blockchain) + // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) @@ -133,7 +135,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { } leth.txPool = light.NewTxPool(leth.chainConfig, leth.blockchain, leth.relay) - if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, true, ClientProtocolVersions, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, leth.serverPool, quitSync, &leth.wg); err != nil { + if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, true, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, leth.serverPool, quitSync, &leth.wg); err != nil { return nil, err } leth.ApiBackend = &LesApiBackend{leth, nil} @@ -215,14 +217,14 @@ func (s *LightEthereum) ResetWithGenesisBlock(gb *types.Block) { func (s *LightEthereum) BlockChain() *light.LightChain { return s.blockchain } func (s *LightEthereum) TxPool() *light.TxPool { return s.txPool } func (s *LightEthereum) Engine() consensus.Engine { return s.engine } -func (s *LightEthereum) LesVersion() int { return int(s.protocolManager.SubProtocols[0].Version) } +func (s *LightEthereum) LesVersion() int { return int(ClientProtocolVersions[0]) } func (s *LightEthereum) Downloader() *downloader.Downloader { return s.protocolManager.downloader } func (s *LightEthereum) EventMux() *event.TypeMux { return s.eventMux } // Protocols implements node.Service, returning all the currently configured // network protocols to start. func (s *LightEthereum) Protocols() []p2p.Protocol { - return s.protocolManager.SubProtocols + return s.makeProtocols(ClientProtocolVersions) } // Start implements node.Service, starting all internal goroutines needed by the @@ -242,12 +244,8 @@ func (s *LightEthereum) Start(srvr *p2p.Server) error { // Ethereum protocol. func (s *LightEthereum) Stop() error { s.odr.Stop() - if s.bloomIndexer != nil { - s.bloomIndexer.Close() - } - if s.chtIndexer != nil { - s.chtIndexer.Close() - } + s.bloomIndexer.Close() + s.chtIndexer.Close() s.blockchain.Stop() s.protocolManager.Stop() s.txPool.Stop() diff --git a/les/commons.go b/les/commons.go new file mode 100644 index 0000000000..251b7a5833 --- /dev/null +++ b/les/commons.go @@ -0,0 +1,106 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/light" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/params" +) + +// lesCommons contains fields needed by both server and client. +type lesCommons struct { + config *eth.Config + chainDb ethdb.Database + protocolManager *ProtocolManager + chtIndexer, bloomTrieIndexer *core.ChainIndexer +} + +// NodeInfo represents a short summary of the Ethereum sub-protocol metadata +// known about the host peer. +type NodeInfo struct { + Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) + Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain + Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block + Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules + Head common.Hash `json:"head"` // SHA3 hash of the host's best owned block + CHT light.TrustedCheckpoint `json:"cht"` // Trused CHT checkpoint for fast catchup +} + +// makeProtocols creates protocol descriptors for the given LES versions. +func (c *lesCommons) makeProtocols(versions []uint) []p2p.Protocol { + protos := make([]p2p.Protocol, len(versions)) + for i, version := range versions { + version := version + protos[i] = p2p.Protocol{ + Name: "les", + Version: version, + Length: ProtocolLengths[version], + NodeInfo: c.nodeInfo, + Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { + return c.protocolManager.runPeer(version, p, rw) + }, + PeerInfo: func(id discover.NodeID) interface{} { + if p := c.protocolManager.peers.Peer(fmt.Sprintf("%x", id[:8])); p != nil { + return p.Info() + } + return nil + }, + } + } + return protos +} + +// nodeInfo retrieves some protocol metadata about the running host node. +func (c *lesCommons) nodeInfo() interface{} { + var cht light.TrustedCheckpoint + sections, _, sectionHead := c.chtIndexer.Sections() + sections2, _, sectionHead2 := c.bloomTrieIndexer.Sections() + if sections2 < sections { + sections = sections2 + sectionHead = sectionHead2 + } + if sections > 0 { + sectionIndex := sections - 1 + cht = light.TrustedCheckpoint{ + SectionIdx: sectionIndex, + SectionHead: sectionHead, + CHTRoot: light.GetChtRoot(c.chainDb, sectionIndex, sectionHead), + BloomRoot: light.GetBloomTrieRoot(c.chainDb, sectionIndex, sectionHead), + } + } + + chain := c.protocolManager.blockchain + head := chain.CurrentHeader() + hash := head.Hash() + return &NodeInfo{ + Network: c.config.NetworkId, + Difficulty: chain.GetTd(hash, head.Number.Uint64()), + Genesis: chain.Genesis().Hash(), + Config: chain.Config(), + Head: chain.CurrentHeader().Hash(), + CHT: cht, + } +} diff --git a/les/handler.go b/les/handler.go index ccb4a88448..ca40eaabfa 100644 --- a/les/handler.go +++ b/les/handler.go @@ -20,7 +20,6 @@ package les import ( "encoding/binary" "encoding/json" - "errors" "fmt" "math/big" "net" @@ -40,7 +39,6 @@ import ( "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" @@ -65,10 +63,6 @@ const ( disableClientRemovePeer = false ) -// errIncompatibleConfig is returned if the requested protocols and configs are -// not compatible (low protocol version restrictions and high requirements). -var errIncompatibleConfig = errors.New("incompatible configuration") - func errResp(code errCode, format string, v ...interface{}) error { return fmt.Errorf("%v - %v", code, fmt.Sprintf(format, v...)) } @@ -115,8 +109,6 @@ type ProtocolManager struct { peers *peerSet maxPeers int - SubProtocols []p2p.Protocol - eventMux *event.TypeMux // channels for fetcher, syncer, txsyncLoop @@ -131,7 +123,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, protocolVersions []uint, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, serverPool *serverPool, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { +func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, serverPool *serverPool, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ lightSync: lightSync, @@ -155,54 +147,6 @@ func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, protoco manager.reqDist = odr.retriever.dist } - // Initiate a sub-protocol for every implemented version we can handle - manager.SubProtocols = make([]p2p.Protocol, 0, len(protocolVersions)) - for _, version := range protocolVersions { - // Compatible, initialize the sub-protocol - version := version // Closure for the run - manager.SubProtocols = append(manager.SubProtocols, p2p.Protocol{ - Name: "les", - Version: version, - Length: ProtocolLengths[version], - Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - var entry *poolEntry - peer := manager.newPeer(int(version), networkId, p, rw) - if manager.serverPool != nil { - addr := p.RemoteAddr().(*net.TCPAddr) - entry = manager.serverPool.connect(peer, addr.IP, uint16(addr.Port)) - } - peer.poolEntry = entry - select { - case manager.newPeerCh <- peer: - manager.wg.Add(1) - defer manager.wg.Done() - err := manager.handle(peer) - if entry != nil { - manager.serverPool.disconnect(entry) - } - return err - case <-manager.quitSync: - if entry != nil { - manager.serverPool.disconnect(entry) - } - return p2p.DiscQuitting - } - }, - NodeInfo: func() interface{} { - return manager.NodeInfo() - }, - PeerInfo: func(id discover.NodeID) interface{} { - if p := manager.peers.Peer(fmt.Sprintf("%x", id[:8])); p != nil { - return p.Info() - } - return nil - }, - }) - } - if len(manager.SubProtocols) == 0 { - return nil, errIncompatibleConfig - } - removePeer := manager.removePeer if disableClientRemovePeer { removePeer = func(id string) {} @@ -262,6 +206,32 @@ func (pm *ProtocolManager) Stop() { log.Info("Light Ethereum protocol stopped") } +// runPeer is the p2p protocol run function for the given version. +func (pm *ProtocolManager) runPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) error { + var entry *poolEntry + peer := pm.newPeer(int(version), pm.networkId, p, rw) + if pm.serverPool != nil { + addr := p.RemoteAddr().(*net.TCPAddr) + entry = pm.serverPool.connect(peer, addr.IP, uint16(addr.Port)) + } + peer.poolEntry = entry + select { + case pm.newPeerCh <- peer: + pm.wg.Add(1) + defer pm.wg.Done() + err := pm.handle(peer) + if entry != nil { + pm.serverPool.disconnect(entry) + } + return err + case <-pm.quitSync: + if entry != nil { + pm.serverPool.disconnect(entry) + } + return p2p.DiscQuitting + } +} + func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { return newPeer(pv, nv, p, newMeteredMsgWriter(rw)) } @@ -1203,50 +1173,6 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { return stats } -// NodeInfo represents a short summary of the Ethereum sub-protocol metadata -// known about the host peer. -type NodeInfo struct { - Network uint64 `json:"network"` // Ethereum network ID (1=Frontier, 2=Morden, Ropsten=3, Rinkeby=4) - Difficulty *big.Int `json:"difficulty"` // Total difficulty of the host's blockchain - Genesis common.Hash `json:"genesis"` // SHA3 hash of the host's genesis block - Config *params.ChainConfig `json:"config"` // Chain configuration for the fork rules - Head common.Hash `json:"head"` // SHA3 hash of the host's best owned block - CHT light.TrustedCheckpoint `json:"cht"` // Trused CHT checkpoint for fast catchup -} - -// NodeInfo retrieves some protocol metadata about the running host node. -func (self *ProtocolManager) NodeInfo() *NodeInfo { - head := self.blockchain.CurrentHeader() - hash := head.Hash() - - var cht light.TrustedCheckpoint - - sections, _, sectionHead := self.odr.ChtIndexer().Sections() - sections2, _, sectionHead2 := self.odr.BloomTrieIndexer().Sections() - if sections2 < sections { - sections = sections2 - sectionHead = sectionHead2 - } - if sections > 0 { - sectionIndex := sections - 1 - cht = light.TrustedCheckpoint{ - SectionIdx: sectionIndex, - SectionHead: sectionHead, - CHTRoot: light.GetChtRoot(self.chainDb, sectionIndex, sectionHead), - BloomRoot: light.GetBloomTrieRoot(self.chainDb, sectionIndex, sectionHead), - } - } - - return &NodeInfo{ - Network: self.networkId, - Difficulty: self.blockchain.GetTd(hash, head.Number.Uint64()), - Genesis: self.blockchain.Genesis().Hash(), - Config: self.blockchain.Config(), - Head: hash, - CHT: cht, - } -} - // downloaderPeerNotify implements peerSetNotify type downloaderPeerNotify ProtocolManager diff --git a/les/helper_test.go b/les/helper_test.go index 50c97e06e1..8817c20c7d 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -172,18 +172,12 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor chain = blockchain } - var protocolVersions []uint - if lightSync { - protocolVersions = ClientProtocolVersions - } else { - protocolVersions = ServerProtocolVersions - } - pm, err := NewProtocolManager(gspec.Config, lightSync, protocolVersions, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup)) + pm, err := NewProtocolManager(gspec.Config, lightSync, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup)) if err != nil { return nil, err } if !lightSync { - srv := &LesServer{protocolManager: pm} + srv := &LesServer{lesCommons: lesCommons{protocolManager: pm}} pm.server = srv srv.defParams = &flowcontrol.ServerParams{ diff --git a/les/server.go b/les/server.go index a934fbf26e..df98d1e3a8 100644 --- a/les/server.go +++ b/les/server.go @@ -38,21 +38,19 @@ import ( ) type LesServer struct { - config *eth.Config - protocolManager *ProtocolManager - fcManager *flowcontrol.ClientManager // nil if our node is client only - fcCostStats *requestCostStats - defParams *flowcontrol.ServerParams - lesTopics []discv5.Topic - privateKey *ecdsa.PrivateKey - quitSync chan struct{} + lesCommons - chtIndexer, bloomTrieIndexer *core.ChainIndexer + fcManager *flowcontrol.ClientManager // nil if our node is client only + fcCostStats *requestCostStats + defParams *flowcontrol.ServerParams + lesTopics []discv5.Topic + privateKey *ecdsa.PrivateKey + quitSync chan struct{} } func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { quitSync := make(chan struct{}) - pm, err := NewProtocolManager(eth.BlockChain().Config(), false, ServerProtocolVersions, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, nil, quitSync, new(sync.WaitGroup)) + pm, err := NewProtocolManager(eth.BlockChain().Config(), false, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, nil, quitSync, new(sync.WaitGroup)) if err != nil { return nil, err } @@ -63,13 +61,17 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { } srv := &LesServer{ - config: config, - protocolManager: pm, - quitSync: quitSync, - lesTopics: lesTopics, - chtIndexer: light.NewChtIndexer(eth.ChainDb(), false, nil), - bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), false, nil), + lesCommons: lesCommons{ + config: config, + chainDb: eth.ChainDb(), + chtIndexer: light.NewChtIndexer(eth.ChainDb(), false, nil), + bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), false, nil), + protocolManager: pm, + }, + quitSync: quitSync, + lesTopics: lesTopics, } + logger := log.New() chtV1SectionCount, _, _ := srv.chtIndexer.Sections() // indexer still uses LES/1 4k section size for backwards server compatibility @@ -104,7 +106,7 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { } func (s *LesServer) Protocols() []p2p.Protocol { - return s.protocolManager.SubProtocols + return s.makeProtocols(ServerProtocolVersions) } // Start starts the LES server From 22cd3f70a68da31b1054fcfca74d187133df29b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 17 Aug 2018 15:47:14 +0300 Subject: [PATCH 22/96] miner: update mining log with correct fee calculation --- miner/worker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index e7e2796458..2f76f2a92c 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -782,8 +782,8 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st w.unconfirmed.Shift(block.NumberU64() - 1) feesWei := new(big.Int) - for _, tx := range block.Transactions() { - feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(tx.Gas()), tx.GasPrice())) + for i, tx := range block.Transactions() { + feesWei.Add(feesWei, new(big.Int).Mul(new(big.Int).SetUint64(receipts[i].GasUsed), tx.GasPrice())) } feesEth := new(big.Float).Quo(new(big.Float).SetInt(feesWei), new(big.Float).SetInt(big.NewInt(params.Ether))) From 251c868008f30b2991bc6986e60a0e7bbdc78b05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 17 Aug 2018 18:12:39 +0300 Subject: [PATCH 23/96] consensus/ethash: reduce notify test aggressiveness --- consensus/ethash/sealer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go index 6d8a770495..6c7157a5ae 100644 --- a/consensus/ethash/sealer_test.go +++ b/consensus/ethash/sealer_test.go @@ -70,7 +70,7 @@ func TestRemoteNotify(t *testing.T) { // issues in the notifications. func TestRemoteMultiNotify(t *testing.T) { // Start a simple webserver to capture notifications - sink := make(chan [3]string, 1024) + sink := make(chan [3]string, 64) server := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { From 0fd02fe9cff54a5edce87588ed780c76a95329fd Mon Sep 17 00:00:00 2001 From: hackyminer Date: Sat, 18 Aug 2018 06:07:20 +0900 Subject: [PATCH 24/96] console: fixed comment typo --- console/console.go | 2 +- console/console_test.go | 8 ++++---- console/prompter.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/console/console.go b/console/console.go index 56e03837ac..3c397f8006 100644 --- a/console/console.go +++ b/console/console.go @@ -314,7 +314,7 @@ func (c *Console) Interactive() { input = "" // Current user input scheduler = make(chan string) // Channel to send the next prompt on and receive the input ) - // Start a goroutine to listen for promt requests and send back inputs + // Start a goroutine to listen for prompt requests and send back inputs go func() { for { // Read the next user input diff --git a/console/console_test.go b/console/console_test.go index 7b1629c032..26465ca6f4 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -201,7 +201,7 @@ func TestInteractive(t *testing.T) { go tester.console.Interactive() - // Wait for a promt and send a statement back + // Wait for a prompt and send a statement back select { case <-tester.input.scheduler: case <-time.After(time.Second): @@ -212,7 +212,7 @@ func TestInteractive(t *testing.T) { case <-time.After(time.Second): t.Fatalf("input feedback timeout") } - // Wait for the second promt and ensure first statement was evaluated + // Wait for the second prompt and ensure first statement was evaluated select { case <-tester.input.scheduler: case <-time.After(time.Second): @@ -249,7 +249,7 @@ func TestExecute(t *testing.T) { } // Tests that the JavaScript objects returned by statement executions are properly -// pretty printed instead of just displaing "[object]". +// pretty printed instead of just displaying "[object]". func TestPrettyPrint(t *testing.T) { tester := newTester(t, nil) defer tester.Close(t) @@ -300,7 +300,7 @@ func TestIndenting(t *testing.T) { }{ {`var a = 1;`, 0}, {`"some string"`, 0}, - {`"some string with (parentesis`, 0}, + {`"some string with (parenthesis`, 0}, {`"some string with newline ("`, 0}, {`function v(a,b) {}`, 0}, diff --git a/console/prompter.go b/console/prompter.go index c477b48178..9b90034db9 100644 --- a/console/prompter.go +++ b/console/prompter.go @@ -27,7 +27,7 @@ import ( // Only this reader may be used for input because it keeps an internal buffer. var Stdin = newTerminalPrompter() -// UserPrompter defines the methods needed by the console to promt the user for +// UserPrompter defines the methods needed by the console to prompt the user for // various types of inputs. type UserPrompter interface { // PromptInput displays the given prompt to the user and requests some textual From d3488c1affee6843d75db77b36f504c73e5e02e7 Mon Sep 17 00:00:00 2001 From: Wuxiang Date: Mon, 20 Aug 2018 20:07:21 +0800 Subject: [PATCH 25/96] p2p: fix typo (#17446) --- p2p/protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/protocol.go b/p2p/protocol.go index ee747ba23d..948aeb494f 100644 --- a/p2p/protocol.go +++ b/p2p/protocol.go @@ -35,7 +35,7 @@ type Protocol struct { // by the protocol. Length uint64 - // Run is called in a new groutine when the protocol has been + // Run is called in a new goroutine when the protocol has been // negotiated with a peer. It should read and write messages from // rw. The Payload for each message must be fully consumed. // From c4078fc80527b2cf0c2fd3e8771e9db06a597152 Mon Sep 17 00:00:00 2001 From: Elad Date: Mon, 20 Aug 2018 14:09:50 +0200 Subject: [PATCH 26/96] cmd/swarm: added swarm bootnodes (#17414) --- cmd/swarm/bootnodes.go | 77 ++++++++++++++++++++++++++++++++++++++++++ cmd/swarm/config.go | 8 ----- cmd/swarm/main.go | 51 ++++++++++++---------------- swarm/api/config.go | 2 -- 4 files changed, 98 insertions(+), 40 deletions(-) create mode 100644 cmd/swarm/bootnodes.go diff --git a/cmd/swarm/bootnodes.go b/cmd/swarm/bootnodes.go new file mode 100644 index 0000000000..cbba9970da --- /dev/null +++ b/cmd/swarm/bootnodes.go @@ -0,0 +1,77 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +var SwarmBootnodes = []string{ + // Foundation Swarm Gateway Cluster + "enode://e5c6f9215c919a5450a7b8c14c22535607b69f2c8e1e7f6f430cb25d7a2c27cd1df4c4f18ad7c1d7e5162e271ffcd3f20b1a1467fb6e790e7d727f3b2193de97@52.232.7.187:30399", + "enode://9b2fe07e69ccc7db5fef15793dab7d7d2e697ed92132d6e9548218e68a34613a8671ad03a6658d862b468ed693cae8a0f8f8d37274e4a657ffb59ca84676e45b@52.232.7.187:30400", + "enode://76c1059162c93ef9df0f01097c824d17c492634df211ef4c806935b349082233b63b90c23970254b3b7138d630400f7cf9b71e80355a446a8b733296cb04169a@52.232.7.187:30401", + "enode://ce46bbe2a8263145d65252d52da06e000ad350ed09c876a71ea9544efa42f63c1e1b6cc56307373aaad8f9dd069c90d0ed2dd1530106200e16f4ca681dd8ae2d@52.232.7.187:30402", + "enode://f431e0d6008a6c35c6e670373d828390c8323e53da8158e7bfc43cf07e632cc9e472188be8df01decadea2d4a068f1428caba769b632554a8fb0607bc296988f@52.232.7.187:30403", + "enode://174720abfff83d7392f121108ae50ea54e04889afe020df883655c0f6cb95414db945a0228d8982fe000d86fc9f4b7669161adc89cd7cd56f78f01489ab2b99b@52.232.7.187:30404", + "enode://2ae89be4be61a689b6f9ecee4360a59e185e010ab750f14b63b4ae43d4180e872e18e3437d4386ce44875dc7cc6eb761acba06412fe3178f3dac1dab3b65703e@52.232.7.187:30405", + "enode://24abebe1c0e6d75d6052ce3219a87be8573fd6397b4cb51f0773b83abba9b3d872bfb273cdc07389715b87adfac02f5235f5241442c5089802cbd8d42e310fce@52.232.7.187:30406", + "enode://d08dfa46bfbbdbcaafbb6e34abee4786610f6c91e0b76d7881f0334ac10dda41d8c1f2b6eedffb4493293c335c0ad46776443b2208d1fbbb9e1a90b25ee4eef2@52.232.7.187:30407", + "enode://8d95eb0f837d27581a43668ed3b8783d69dc4e84aa3edd7a0897e026155c8f59c8702fdc0375ee7bac15757c9c78e1315d9b73e4ce59c936db52ea4ae2f501c7@52.232.7.187:30408", + "enode://a5967cc804aebd422baaaba9f06f27c9e695ccab335b61088130f8cbe64e3cdf78793868c7051dfc06eecfe844fad54bc7f6dfaed9db3c7ecef279cb829c25fb@52.232.7.187:30409", + "enode://5f00134d81a8f2ebcc46f8766f627f492893eda48138f811b7de2168308171968f01710bca6da05764e74f14bae41652f554e6321f1aed85fa3461e89d075dbf@52.232.7.187:30410", + "enode://b2142b79b01a5aa66a5e23cc35e78219a8e97bc2412a6698cee24ae02e87078b725d71730711bd62e25ff1aa8658c6633778af8ac14c63814a337c3dd0ebda9f@52.232.7.187:30411", + "enode://1ffa7651094867d6486ce3ef46d27a052c2cb968b618346c6df7040322c7efc3337547ba85d4cbba32e8b31c42c867202554735c06d4c664b9afada2ed0c4b3c@52.232.7.187:30412", + "enode://129e0c3d5f5df12273754f6f703d2424409fa4baa599e0b758c55600169313887855e75b082028d2302ec034b303898cd697cc7ae8256ba924ce927510da2c8d@52.232.7.187:30413", + "enode://419e2dc0d2f5b022cf16b0e28842658284909fa027a0fbbb5e2b755e7f846ea02a8f0b66a7534981edf6a7bcf8a14855344c6668e2cd4476ccd35a11537c9144@52.232.7.187:30414", + "enode://23d55ad900583231b91f2f62e3f72eb498b342afd58b682be3af052eed62b5651094471065981de33d8786f075f05e3cca499503b0ac8ae84b2a06e99f5b0723@52.232.7.187:30415", + "enode://bc56e4158c00e9f616d7ea533def20a89bef959df4e62a768ff238ff4e1e9223f57ecff969941c20921bad98749baae311c0fbebce53bf7bbb9d3dc903640990@52.232.7.187:30416", + "enode://433ce15199c409875e7e72fffd69fdafe746f17b20f0d5555281722a65fde6c80328fab600d37d8624509adc072c445ce0dad4a1c01cff6acf3132c11d429d4d@52.232.7.187:30417", + "enode://632ee95b8f0eac51ef89ceb29313fef3a60050181d66a6b125583b1a225a7694b252edc016efb58aa3b251da756cb73280842a022c658ed405223b2f58626343@52.232.7.187:30418", + "enode://4a0f9bcff7a4b9ee453fb298d0fb222592efe121512e30cd72fef631beb8c6a15153a1456eb073ee18551c0e003c569651a101892dc4124e90b933733a498bb5@52.232.7.187:30419", + "enode://f0d80fbc72d16df30e19aac3051eb56a7aff0c8367686702e01ea132d8b0b3ee00cadd6a859d2cca98ec68d3d574f8a8a87dba2347ec1e2818dc84bc3fa34fae@52.232.7.187:30420", + "enode://a199146906e4f9f2b94b195a8308d9a59a3564b92efaab898a4243fe4c2ad918b7a8e4853d9d901d94fad878270a2669d644591299c3d43de1b298c00b92b4a7@52.232.7.187:30421", + "enode://052036ea8736b37adbfb684d90ce43e11b3591b51f31489d7c726b03618dea4f73b1e659deb928e6bf40564edcdcf08351643f42db3d4ca1c2b5db95dad59e94@52.232.7.187:30422", + "enode://460e2b8c6da8f12fac96c836e7d108f4b7ec55a1c64631bb8992339e117e1c28328fee83af863196e20af1487a655d13e5ceba90e980e92502d5bac5834c1f71@52.232.7.187:30423", + "enode://6d2cdd13741b2e72e9031e1b93c6d9a4e68de2844aa4e939f6a8a8498a7c1d7e2ee4c64217e92a6df08c9a32c6764d173552810ef1bd2ecb356532d389dd2136@52.232.7.187:30424", + "enode://62105fc25ce2cd5b299647f47eaa9211502dc76f0e9f461df915782df7242ac3223e3db04356ae6ed2977ccac20f0b16864406e9ca514a40a004cb6a5d0402aa@52.232.7.187:30425", + "enode://e0e388fc520fd493c33f0ce16685e6f98fb6aec28f2edc14ee6b179594ee519a896425b0025bb6f0e182dd3e468443f19c70885fbc66560d000093a668a86aa8@52.232.7.187:30426", + "enode://63f3353a72521ea10022127a4fe6b4acbef197c3fe668fd9f4805542d8a6fcf79f6335fbab62d180a35e19b739483e740858b113fdd7c13a26ad7b4e318a5aef@52.232.7.187:30427", + "enode://33a42b927085678d4aefd4e70b861cfca6ef5f6c143696c4f755973fd29e64c9e658cad57a66a687a7a156da1e3688b1fbdd17bececff2ee009fff038fa5666b@52.232.7.187:30428", + "enode://259ab5ab5c1daee3eab7e3819ab3177b82d25c29e6c2444fdd3f956e356afae79a72840ccf2d0665fe82c81ebc3b3734da1178ac9fd5d62c67e674b69f86b6be@52.232.7.187:30429", + "enode://558bccad7445ce3fd8db116ed6ab4aed1324fdbdac2348417340c1764dc46d46bffe0728e5b7d5c36f12e794c289f18f57f08f085d2c65c9910a5c7a65b6a66a@52.232.7.187:30430", + "enode://abe60937a0657ffded718e3f84a32987286983be257bdd6004775c4b525747c2b598f4fac49c8de324de5ce75b22673fa541a7ce2d555fb7f8ca325744ae3577@52.232.7.187:30431", + "enode://bce6f0aaa5b230742680084df71d4f026b3eff7f564265599216a1b06b765303fdc9325de30ffd5dfdaf302ce4b14322891d2faea50ce2ca298d7409f5858339@52.232.7.187:30432", + "enode://21b957c4e03277d42be6660730ec1b93f540764f26c6abdb54d006611139c7081248486206dfbf64fcaffd62589e9c6b8ea77a5297e4b21a605f1bcf49483ed0@52.232.7.187:30433", + "enode://ff104e30e64f24c3d7328acee8b13354e5551bc8d60bb25ecbd9632d955c7e34bb2d969482d173355baad91c8282f8b592624eb3929151090da3b4448d4d58fb@52.232.7.187:30434", + "enode://c76e2b5f81a521bceaec1518926a21380a345df9cf463461562c6845795512497fb67679e155fc96a74350f8b78de8f4c135dd52b106dbbb9795452021d09ea5@52.232.7.187:30435", + "enode://3288fd860105164f3e9b69934c4eb18f7146cfab31b5a671f994e21a36e9287766e5f9f075aefbc404538c77f7c2eb2a4495020a7633a1c3970d94e9fa770aeb@52.232.7.187:30436", + "enode://6cea859c7396d46b20cfcaa80f9a11cd112f8684f2f782f7b4c0e1e0af9212113429522075101923b9b957603e6c32095a6a07b5e5e35183c521952ee108dfaf@52.232.7.187:30437", + "enode://f628ec56e4ca8317cc24cc4ac9b27b95edcce7b96e1c7f3b53e30de4a8580fe44f2f0694a513bdb0a431acaf2824074d6ace4690247bbc34c14f426af8c056ea@52.232.7.187:30438", + "enode://055ec8b26fc105c4f97970a1cce9773a5e34c03f511b839db742198a1c571e292c54aa799e9afb991cc8a560529b8cdf3e0c344bc6c282aff2f68eec59361ddf@52.232.7.187:30439", + "enode://48cb0d430c328974226aa33a931d8446cd5a8d40f3ead8f4ce7ad60faa1278192eb6d58bed91258d63e81f255fc107eec2425ce2ae8b22350dd556076e160610@52.232.7.187:30440", + "enode://3fadb7af7f770d5ffc6b073b8d42834bebb18ce1fe8a4fe270d2b799e7051327093960dc61d9a18870db288f7746a0e6ea2a013cd6ab0e5f97ca08199473aace@52.232.7.187:30441", + "enode://a5d7168024c9992769cf380ffa559a64b4f39a29d468f579559863814eb0ae0ed689ac0871a3a2b4c78b03297485ec322d578281131ef5d5c09a4beb6200a97a@52.232.7.187:30442", + "enode://9c57744c5b2c2d71abcbe80512652f9234d4ab041b768a2a886ab390fe6f184860f40e113290698652d7e20a8ac74d27ac8671db23eb475b6c5e6253e4693bf8@52.232.7.187:30443", + "enode://daca9ff0c3176045a0e0ed228dee00ec86bc0939b135dc6b1caa23745d20fd0332e1ee74ad04020e89df56c7146d831a91b89d15ca3df05ba7618769fefab376@52.232.7.187:30444", + "enode://a3f6af59428cb4b9acb198db15ef5554fa43c2b0c18e468a269722d64a27218963a2975eaf82750b6262e42192b5e3669ea51337b4cda62b33987981bc5e0c1a@52.232.7.187:30445", + "enode://fe571422fa4651c3354c85dac61911a6a6520dd3c0332967a49d4133ca30e16a8a4946fa73ca2cb5de77917ea701a905e1c3015b2f4defcd53132b61cc84127a@52.232.7.187:30446", + + // Mainframe + "enode://ee9a5a571ea6c8a59f9a8bb2c569c865e922b41c91d09b942e8c1d4dd2e1725bd2c26149da14de1f6321a2c6fdf1e07c503c3e093fb61696daebf74d6acd916b@54.186.219.160:30399", + "enode://a03f0562ecb8a992ad5242345535e73483cdc18ab934d36bf24b567d43447c2cea68f89f1d51d504dd13acc30f24ebce5a150bea2ccb1b722122ce4271dc199d@52.67.248.147:30399", + "enode://e2cbf9eafd85903d3b1c56743035284320695e0072bc8d7396e0542aa5e1c321b236f67eab66b79c2f15d4447fa4bbe74dd67d0467da23e7eb829f60ec8a812b@13.58.169.1:30399", + "enode://8b8c6bda6047f1cad9fab2db4d3d02b7aa26279902c32879f7bcd4a7d189fee77fdc36ee151ce6b84279b4792e72578fd529d2274d014132465758fbfee51cee@13.209.13.15:30399", + "enode://63f6a8818927e429585287cf2ca0cb9b11fa990b7b9b331c2962cdc6f21807a2473b26e8256225c26caff70d7218e59586d704d49061452c6852e382c885d03c@35.154.106.174:30399", + "enode://ed4bd3b794ed73f18e6dcc70c6624dfec63b5654f6ab54e8f40b16eff8afbd342d4230e099ddea40e84423f81b2d2ea79799dc345257b1fec6f6c422c9d008f7@52.213.20.99:30399", +} diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 1183f8bc81..ae4b5816e6 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -233,10 +233,6 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.Cors = cors } - if ctx.GlobalIsSet(utils.BootnodesFlag.Name) { - currentConfig.BootNodes = ctx.GlobalString(utils.BootnodesFlag.Name) - } - if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" { currentConfig.LocalStoreParams.ChunkDbPath = storePath } @@ -334,10 +330,6 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { currentConfig.Cors = cors } - if bootnodes := os.Getenv(SWARM_ENV_BOOTNODES); bootnodes != "" { - currentConfig.BootNodes = bootnodes - } - return currentConfig } diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 76be60cb68..637ae06e96 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -37,7 +37,6 @@ import ( "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" @@ -67,14 +66,7 @@ OPTIONS: ` var ( - gitCommit string // Git SHA1 commit hash of the release (set via linker flags) - testbetBootNodes = []string{ - "enode://ec8ae764f7cb0417bdfb009b9d0f18ab3818a3a4e8e7c67dd5f18971a93510a2e6f43cd0b69a27e439a9629457ea804104f37c85e41eed057d3faabbf7744cdf@13.74.157.139:30429", - "enode://c2e1fceb3bf3be19dff71eec6cccf19f2dbf7567ee017d130240c670be8594bc9163353ca55dd8df7a4f161dd94b36d0615c17418b5a3cdcbb4e9d99dfa4de37@13.74.157.139:30430", - "enode://fe29b82319b734ce1ec68b84657d57145fee237387e63273989d354486731e59f78858e452ef800a020559da22dcca759536e6aa5517c53930d29ce0b1029286@13.74.157.139:30431", - "enode://1d7187e7bde45cf0bee489ce9852dd6d1a0d9aa67a33a6b8e6db8a4fbc6fcfa6f0f1a5419343671521b863b187d1c73bad3603bae66421d157ffef357669ddb8@13.74.157.139:30432", - "enode://0e4cba800f7b1ee73673afa6a4acead4018f0149d2e3216be3f133318fd165b324cd71b81fbe1e80deac8dbf56e57a49db7be67f8b9bc81bd2b7ee496434fb5d@13.74.157.139:30433", - } + gitCommit string // Git SHA1 commit hash of the release (set via linker flags) ) var ( @@ -619,6 +611,9 @@ func bzzd(ctx *cli.Context) error { if _, err := os.Stat(bzzconfig.Path); err == nil { cfg.DataDir = bzzconfig.Path } + + //optionally set the bootnodes before configuring the node + setSwarmBootstrapNodes(ctx, &cfg) //setup the ethereum node utils.SetNodeConfig(ctx, &cfg) stack, err := node.New(&cfg) @@ -643,16 +638,6 @@ func bzzd(ctx *cli.Context) error { stack.Stop() }() - // Add bootnodes as initial peers. - if bzzconfig.BootNodes != "" { - bootnodes := strings.Split(bzzconfig.BootNodes, ",") - injectBootnodes(stack.Server(), bootnodes) - } else { - if bzzconfig.NetworkID == 3 { - injectBootnodes(stack.Server(), testbetBootNodes) - } - } - stack.Wait() return nil } @@ -760,17 +745,6 @@ func getPassPhrase(prompt string, i int, passwords []string) string { return password } -func injectBootnodes(srv *p2p.Server, nodes []string) { - for _, url := range nodes { - n, err := discover.ParseNode(url) - if err != nil { - log.Error("Invalid swarm bootnode", "err", err) - continue - } - srv.AddPeer(n) - } -} - // addDefaultHelpSubcommand scans through defined CLI commands and adds // a basic help subcommand to each // if a help command is already defined, it will take precedence over the default. @@ -783,3 +757,20 @@ func addDefaultHelpSubcommands(commands []cli.Command) { } } } + +func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) { + if ctx.GlobalIsSet(utils.BootnodesFlag.Name) || ctx.GlobalIsSet(utils.BootnodesV4Flag.Name) { + return + } + + cfg.P2P.BootstrapNodes = []*discover.Node{} + + for _, url := range SwarmBootnodes { + node, err := discover.ParseNode(url) + if err != nil { + log.Error("Bootstrap URL invalid", "enode", url, "err", err) + } + cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node) + } + log.Debug("added default swarm bootnodes", "length", len(cfg.P2P.BootstrapNodes)) +} diff --git a/swarm/api/config.go b/swarm/api/config.go index bdfffdd05f..3044dc2e52 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -68,7 +68,6 @@ type Config struct { SwapAPI string Cors string BzzAccount string - BootNodes string privateKey *ecdsa.PrivateKey } @@ -93,7 +92,6 @@ func NewConfig() (c *Config) { DeliverySkipCheck: false, SyncUpdateDelay: 15 * time.Second, SwapAPI: "", - BootNodes: "", } return From a8aa89accb0bcd4f24cd430d8f100e0ff9e239d0 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 20 Aug 2018 14:10:30 +0200 Subject: [PATCH 27/96] swarm/storage: cleanup task - remove bigger chunks (#17424) --- swarm/storage/ldbstore.go | 67 ++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 7920ee7674..b95aa13b09 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock" "github.com/syndtr/goleveldb/leveldb" @@ -384,14 +385,13 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) { } func (s *LDBStore) Cleanup() { - //Iterates over the database and checks that there are no faulty chunks + //Iterates over the database and checks that there are no chunks bigger than 4kb + var errorsFound, removed, total int + it := s.db.NewIterator() - startPosition := []byte{keyIndex} - it.Seek(startPosition) - var key []byte - var errorsFound, total int - for it.Valid() { - key = it.Key() + defer it.Release() + for ok := it.Seek([]byte{keyIndex}); ok; ok = it.Next() { + key := it.Key() if (key == nil) || (key[0] != keyIndex) { break } @@ -399,27 +399,50 @@ func (s *LDBStore) Cleanup() { var index dpaDBIndex err := decodeIndex(it.Value(), &index) if err != nil { - it.Next() + log.Warn("Cannot decode") + errorsFound++ continue } - data, err := s.db.Get(getDataKey(index.Idx, s.po(Address(key[1:])))) + hash := key[1:] + po := s.po(hash) + datakey := getDataKey(index.Idx, po) + data, err := s.db.Get(datakey) if err != nil { - log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err)) - s.delete(index.Idx, getIndexKey(key[1:]), s.po(Address(key[1:]))) - errorsFound++ - } else { - hasher := s.hashfunc() - hasher.Write(data[32:]) - hash := hasher.Sum(nil) - if !bytes.Equal(hash, key[1:]) { - log.Warn(fmt.Sprintf("Found invalid chunk. Hash mismatch. hash=%x, key=%x", hash, key[:])) - s.delete(index.Idx, getIndexKey(key[1:]), s.po(Address(key[1:]))) + found := false + + // highest possible proximity is 255 + for po = 1; po <= 255; po++ { + datakey = getDataKey(index.Idx, po) + data, err = s.db.Get(datakey) + if err == nil { + found = true + break + } + } + + if !found { + log.Warn(fmt.Sprintf("Chunk %x found but count not be accessed with any po", key[:])) + errorsFound++ + continue } } - it.Next() + + c := &Chunk{} + ck := data[:32] + decodeData(data, c) + + cs := int64(binary.LittleEndian.Uint64(c.SData[:8])) + log.Trace("chunk", "key", fmt.Sprintf("%x", key[:]), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.SData), "size", cs) + + if len(c.SData) > chunk.DefaultSize+8 { + log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key[:]), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.SData), "size", cs) + s.delete(index.Idx, getIndexKey(key[1:]), po) + removed++ + errorsFound++ + } } - it.Release() - log.Warn(fmt.Sprintf("Found %v errors out of %v entries", errorsFound, total)) + + log.Warn(fmt.Sprintf("Found %v errors out of %v entries. Removed %v chunks.", errorsFound, total, removed)) } func (s *LDBStore) ReIndex() { From 55d050ccd81d726ce47cfb57ecd9de662129b7f1 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 20 Aug 2018 14:11:08 +0200 Subject: [PATCH 28/96] travis: remove brew update and osxfuse install (#17429) --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index afa9ab503f..3ae88aab6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,8 +30,6 @@ matrix: go: 1.10.x script: - unset -f cd # workaround for https://github.com/travis-ci/travis-ci/issues/8703 - - brew update - - brew cask install osxfuse - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES From c929030e280ecae5e5943fa277240ee7d09d7506 Mon Sep 17 00:00:00 2001 From: Aditya Date: Mon, 20 Aug 2018 05:14:50 -0700 Subject: [PATCH 29/96] core/types: fix docs about protected Vs (#17436) --- core/types/transaction.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index 82af9335ff..9c6e77be98 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -119,7 +119,7 @@ func isProtectedV(V *big.Int) bool { v := V.Uint64() return v != 27 && v != 28 } - // anything not 27 or 28 are considered unprotected + // anything not 27 or 28 is considered protected return true } From 1de9ada4016d7028d1d1529bc7d0676c98ddb5e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 20 Aug 2018 15:49:28 +0200 Subject: [PATCH 30/96] light: new CHTs (#17448) --- light/postprocess.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/light/postprocess.go b/light/postprocess.go index 0b25e1d881..42985aff05 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -61,18 +61,18 @@ type TrustedCheckpoint struct { var ( mainnetCheckpoint = TrustedCheckpoint{ name: "mainnet", - SectionIdx: 179, - SectionHead: common.HexToHash("ae778e455492db1183e566fa0c67f954d256fdd08618f6d5a393b0e24576d0ea"), - CHTRoot: common.HexToHash("646b338f9ca74d936225338916be53710ec84020b89946004a8605f04c817f16"), - BloomRoot: common.HexToHash("d0f978f5dbc86e5bf931d8dd5b2ecbebbda6dc78f8896af6a27b46a3ced0ac25"), + SectionIdx: 187, + SectionHead: common.HexToHash("e6baa034efa31562d71ff23676512dec6562c1ad0301e08843b907e81958c696"), + CHTRoot: common.HexToHash("28001955219719cf06de1b08648969139d123a9835fc760547a1e4dabdabc15a"), + BloomRoot: common.HexToHash("395ca2373fc662720ac6b58b3bbe71f68aa0f38b63b2d3553dd32ff3c51eebc4"), } ropstenCheckpoint = TrustedCheckpoint{ name: "ropsten", - SectionIdx: 107, - SectionHead: common.HexToHash("e1988f95399debf45b873e065e5cd61b416ef2e2e5deec5a6f87c3127086e1ce"), - CHTRoot: common.HexToHash("15cba18e4de0ab1e95e202625199ba30147aec8b0b70384b66ebea31ba6a18e0"), - BloomRoot: common.HexToHash("e00fa6389b2e597d9df52172cd8e936879eed0fca4fa59db99e2c8ed682562f2"), + SectionIdx: 117, + SectionHead: common.HexToHash("9529b38631ae30783f56cbe4c3b9f07575b770ecba4f6e20a274b1e2f40fede1"), + CHTRoot: common.HexToHash("6f48e9f101f1fac98e7d74fbbcc4fda138358271ffd974d40d2506f0308bb363"), + BloomRoot: common.HexToHash("8242342e66e942c0cd893484e6736b9862ceb88b43ca344bb06a8285ac1b6d64"), } ) From 7d38d53ae449c6ec06f7b0579f1a189b02222a60 Mon Sep 17 00:00:00 2001 From: Nilesh Trivedi Date: Mon, 20 Aug 2018 19:24:38 +0530 Subject: [PATCH 31/96] cmd/puppeth: accept ssh identity in the server string (#17407) * cmd/puppeth: Accept identityfile in the server string with fallback to id_rsa * cmd/puppeth: code polishes + fix heath check double ports --- cmd/puppeth/ssh.go | 52 ++++++++++++++++++++--------------- cmd/puppeth/wizard_network.go | 8 +++--- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/cmd/puppeth/ssh.go b/cmd/puppeth/ssh.go index 158261ce05..c507596065 100644 --- a/cmd/puppeth/ssh.go +++ b/cmd/puppeth/ssh.go @@ -45,33 +45,44 @@ type sshClient struct { // dial establishes an SSH connection to a remote node using the current user and // the user's configured private RSA key. If that fails, password authentication -// is fallen back to. The caller may override the login user via user@server:port. +// is fallen back to. server can be a string like user:identity@server:port. func dial(server string, pubkey []byte) (*sshClient, error) { - // Figure out a label for the server and a logger - label := server - if strings.Contains(label, ":") { - label = label[:strings.Index(label, ":")] - } - login := "" + // Figure out username, identity, hostname and port + hostname := "" + hostport := server + username := "" + identity := "id_rsa" // default + if strings.Contains(server, "@") { - login = label[:strings.Index(label, "@")] - label = label[strings.Index(label, "@")+1:] - server = server[strings.Index(server, "@")+1:] + prefix := server[:strings.Index(server, "@")] + if strings.Contains(prefix, ":") { + username = prefix[:strings.Index(prefix, ":")] + identity = prefix[strings.Index(prefix, ":")+1:] + } else { + username = prefix + } + hostport = server[strings.Index(server, "@")+1:] } - logger := log.New("server", label) + if strings.Contains(hostport, ":") { + hostname = hostport[:strings.Index(hostport, ":")] + } else { + hostname = hostport + hostport += ":22" + } + logger := log.New("server", server) logger.Debug("Attempting to establish SSH connection") user, err := user.Current() if err != nil { return nil, err } - if login == "" { - login = user.Username + if username == "" { + username = user.Username } // Configure the supported authentication methods (private key and password) var auths []ssh.AuthMethod - path := filepath.Join(user.HomeDir, ".ssh", "id_rsa") + path := filepath.Join(user.HomeDir, ".ssh", identity) if buf, err := ioutil.ReadFile(path); err != nil { log.Warn("No SSH key, falling back to passwords", "path", path, "err", err) } else { @@ -94,14 +105,14 @@ func dial(server string, pubkey []byte) (*sshClient, error) { } } auths = append(auths, ssh.PasswordCallback(func() (string, error) { - fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", login, server) + fmt.Printf("What's the login password for %s at %s? (won't be echoed)\n> ", username, server) blob, err := terminal.ReadPassword(int(os.Stdin.Fd())) fmt.Println() return string(blob), err })) // Resolve the IP address of the remote server - addr, err := net.LookupHost(label) + addr, err := net.LookupHost(hostname) if err != nil { return nil, err } @@ -109,10 +120,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { return nil, errors.New("no IPs associated with domain") } // Try to dial in to the remote server - logger.Trace("Dialing remote SSH server", "user", login) - if !strings.Contains(server, ":") { - server += ":22" - } + logger.Trace("Dialing remote SSH server", "user", username) keycheck := func(hostname string, remote net.Addr, key ssh.PublicKey) error { // If no public key is known for SSH, ask the user to confirm if pubkey == nil { @@ -139,13 +147,13 @@ func dial(server string, pubkey []byte) (*sshClient, error) { // We have a mismatch, forbid connecting return errors.New("ssh key mismatch, readd the machine to update") } - client, err := ssh.Dial("tcp", server, &ssh.ClientConfig{User: login, Auth: auths, HostKeyCallback: keycheck}) + client, err := ssh.Dial("tcp", hostport, &ssh.ClientConfig{User: username, Auth: auths, HostKeyCallback: keycheck}) if err != nil { return nil, err } // Connection established, return our utility wrapper c := &sshClient{ - server: label, + server: hostname, address: addr[0], pubkey: pubkey, client: client, diff --git a/cmd/puppeth/wizard_network.go b/cmd/puppeth/wizard_network.go index d780c550b1..c0ddcc2a3c 100644 --- a/cmd/puppeth/wizard_network.go +++ b/cmd/puppeth/wizard_network.go @@ -62,14 +62,14 @@ func (w *wizard) manageServers() { } } -// makeServer reads a single line from stdin and interprets it as a hostname to -// connect to. It tries to establish a new SSH session and also executing some -// baseline validations. +// makeServer reads a single line from stdin and interprets it as +// username:identity@hostname to connect to. It tries to establish a +// new SSH session and also executing some baseline validations. // // If connection succeeds, the server is added to the wizards configs! func (w *wizard) makeServer() string { fmt.Println() - fmt.Println("Please enter remote server's address:") + fmt.Println("What is the remote server's address ([username[:identity]@]hostname[:port])?") // Read and dial the server to ensure docker is present input := w.readString() From a6d45a5d00e0f37eab9eba43f1ec60429f0dc120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 20 Aug 2018 18:05:06 +0300 Subject: [PATCH 32/96] crypto/bn256: add missing license file, release wrapper in BSD-3 --- crypto/bn256/LICENSE | 28 ++++++++++++++++++++++++++++ crypto/bn256/bn256_fast.go | 18 +++--------------- crypto/bn256/bn256_fuzz.go | 18 +++--------------- crypto/bn256/bn256_slow.go | 18 +++--------------- crypto/bn256/cloudflare/LICENSE | 27 +++++++++++++++++++++++++++ 5 files changed, 64 insertions(+), 45 deletions(-) create mode 100644 crypto/bn256/LICENSE create mode 100644 crypto/bn256/cloudflare/LICENSE diff --git a/crypto/bn256/LICENSE b/crypto/bn256/LICENSE new file mode 100644 index 0000000000..634e0cb2c3 --- /dev/null +++ b/crypto/bn256/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2018 Péter Szilágyi. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/crypto/bn256/bn256_fast.go b/crypto/bn256/bn256_fast.go index a8dfa8f67d..5c081493b0 100644 --- a/crypto/bn256/bn256_fast.go +++ b/crypto/bn256/bn256_fast.go @@ -1,18 +1,6 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// Copyright 2018 Péter Szilágyi. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. // +build amd64 arm64 diff --git a/crypto/bn256/bn256_fuzz.go b/crypto/bn256/bn256_fuzz.go index f360b0541f..6aa1421170 100644 --- a/crypto/bn256/bn256_fuzz.go +++ b/crypto/bn256/bn256_fuzz.go @@ -1,18 +1,6 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// Copyright 2018 Péter Szilágyi. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. // +build gofuzz diff --git a/crypto/bn256/bn256_slow.go b/crypto/bn256/bn256_slow.go index 61373763b8..47df49d417 100644 --- a/crypto/bn256/bn256_slow.go +++ b/crypto/bn256/bn256_slow.go @@ -1,18 +1,6 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// Copyright 2018 Péter Szilágyi. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. // +build !amd64,!arm64 diff --git a/crypto/bn256/cloudflare/LICENSE b/crypto/bn256/cloudflare/LICENSE new file mode 100644 index 0000000000..6a66aea5ea --- /dev/null +++ b/crypto/bn256/cloudflare/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 106d196ec4a6451efedc60ab15957f231fa85639 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 21 Aug 2018 09:48:53 +0200 Subject: [PATCH 33/96] eth: ensure from= 0 { + return nil, fmt.Errorf("end block (#%d) needs to come after start block (#%d)", end, start) + } return api.traceChain(ctx, from, to, config) } From dcd97c41fa2789f2e642cb52b8e18c5bd7bcc81d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 21 Aug 2018 10:34:10 +0200 Subject: [PATCH 34/96] swarm, swarm/network, swarm/pss: log error and fix logs (#17410) * swarm, swarm/network, swarm/pss: log error and fix logs * swarm/pss: log compressed publickey --- swarm/network/hive.go | 4 ++-- swarm/pss/pss.go | 5 +++-- swarm/swarm.go | 34 ++++++++++++++++------------------ 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/swarm/network/hive.go b/swarm/network/hive.go index a54a17d29d..3660210883 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -102,10 +102,10 @@ func NewHive(params *HiveParams, overlay Overlay, store state.Store) *Hive { // server is used to connect to a peer based on its NodeID or enode URL // these are called on the p2p.Server which runs on the node func (h *Hive) Start(server *p2p.Server) error { - log.Info(fmt.Sprintf("%08x hive starting", h.BaseAddr()[:4])) + log.Info("Starting hive", "baseaddr", fmt.Sprintf("%x", h.BaseAddr()[:4])) // if state store is specified, load peers to prepopulate the overlay address book if h.Store != nil { - log.Info("detected an existing store. trying to load peers") + log.Info("Detected an existing store. trying to load peers") if err := h.loadPeers(); err != nil { log.Error(fmt.Sprintf("%08x hive encoutered an error trying to load peers", h.BaseAddr()[:4])) return err diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 5c060b2486..8459211ddb 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -232,12 +232,13 @@ func (p *Pss) Start(srv *p2p.Server) error { } } }() - log.Debug("Started pss", "public key", common.ToHex(crypto.FromECDSAPub(p.PublicKey()))) + log.Info("Started Pss") + log.Info("Loaded EC keys", "pubkey", common.ToHex(crypto.FromECDSAPub(p.PublicKey())), "secp256", common.ToHex(crypto.CompressPubkey(p.PublicKey()))) return nil } func (p *Pss) Stop() error { - log.Info("pss shutting down") + log.Info("Pss shutting down") close(p.quitC) return nil } diff --git a/swarm/swarm.go b/swarm/swarm.go index a895bdfa55..736cd37de8 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -121,12 +121,10 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e backend: backend, privateKey: config.ShiftPrivateKey(), } - log.Debug(fmt.Sprintf("Setting up Swarm service components")) + log.Debug("Setting up Swarm service components") config.HiveParams.Discovery = true - log.Debug(fmt.Sprintf("-> swarm net store shared access layer to Swarm Chunk Store")) - nodeID, err := discover.HexID(config.NodeID) if err != nil { return nil, err @@ -201,8 +199,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e resourceHandler, } - // setup local store - log.Debug(fmt.Sprintf("Set up local storage")) + log.Debug("Setup local storage") self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run) @@ -216,11 +213,9 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e } self.api = api.NewAPI(self.fileStore, self.dns, resourceHandler, self.privateKey) - // Manifests for Smart Hosting - log.Debug(fmt.Sprintf("-> Web3 virtual server API")) self.sfs = fuse.NewSwarmFS(self.api) - log.Debug("-> Initializing Fuse file system") + log.Debug("Initialized FUSE filesystem") return self, nil } @@ -341,7 +336,7 @@ func (self *Swarm) Start(srv *p2p.Server) error { // update uaddr to correct enode newaddr := self.bzz.UpdateLocalAddr([]byte(srv.Self().String())) - log.Warn("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%s", newaddr.UAddr)) + log.Info("Updated bzz local addr", "oaddr", fmt.Sprintf("%x", newaddr.OAddr), "uaddr", fmt.Sprintf("%s", newaddr.UAddr)) // set chequebook if self.config.SwapEnabled { ctx := context.Background() // The initial setup has no deadline. @@ -354,18 +349,17 @@ func (self *Swarm) Start(srv *p2p.Server) error { log.Debug(fmt.Sprintf("SWAP disabled: no cheque book set")) } - log.Warn(fmt.Sprintf("Starting Swarm service")) + log.Info("Starting bzz service") err := self.bzz.Start(srv) if err != nil { log.Error("bzz failed", "err", err) return err } - log.Info(fmt.Sprintf("Swarm network started on bzz address: %x", self.bzz.Hive.Overlay.BaseAddr())) + log.Info("Swarm network started", "bzzaddr", fmt.Sprintf("%x", self.bzz.Hive.Overlay.BaseAddr())) if self.ps != nil { self.ps.Start(srv) - log.Info("Pss started") } // start swarm http proxy server @@ -373,13 +367,17 @@ func (self *Swarm) Start(srv *p2p.Server) error { addr := net.JoinHostPort(self.config.ListenAddr, self.config.Port) server := httpapi.NewServer(self.api, self.config.Cors) - go server.ListenAndServe(addr) - } + if self.config.Cors != "" { + log.Debug("Swarm HTTP proxy CORS headers", "allowedOrigins", self.config.Cors) + } - log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port)) - - if self.config.Cors != "" { - log.Debug(fmt.Sprintf("Swarm http proxy started with corsdomain: %v", self.config.Cors)) + log.Debug("Starting Swarm HTTP proxy", "port", self.config.Port) + go func() { + err := server.ListenAndServe(addr) + if err != nil { + log.Error("Could not start Swarm HTTP proxy", "err", err.Error()) + } + }() } self.periodicallyUpdateGauges() From c582667c9b88fabfca2908e1a6ef9b3a71daef37 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 21 Aug 2018 10:34:40 +0200 Subject: [PATCH 35/96] swarm/network: bump bzz protocol version (#17449) --- swarm/network/protocol.go | 2 +- swarm/network/protocol_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/network/protocol.go b/swarm/network/protocol.go index 7e7fee8efe..7f7ca5eed5 100644 --- a/swarm/network/protocol.go +++ b/swarm/network/protocol.go @@ -44,7 +44,7 @@ const ( // BzzSpec is the spec of the generic swarm handshake var BzzSpec = &protocols.Spec{ Name: "bzz", - Version: 5, + Version: 6, MaxMsgSize: 10 * 1024 * 1024, Messages: []interface{}{ HandshakeMsg{}, diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index b74b72c68b..c052c536a4 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -31,7 +31,7 @@ import ( ) const ( - TestProtocolVersion = 5 + TestProtocolVersion = 6 TestProtocolNetworkID = 3 ) From 76301ca0517ef2d34e6e2d741b0b24cf0ff98642 Mon Sep 17 00:00:00 2001 From: Pierre Neter Date: Tue, 21 Aug 2018 17:36:38 +0700 Subject: [PATCH 36/96] eth: upgradedb subcommand was dropped (#17464) --- eth/backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/backend.go b/eth/backend.go index 6549cb8a3d..588b782568 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -138,7 +138,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if !config.SkipBcVersionCheck { bcVersion := rawdb.ReadDatabaseVersion(chainDb) if bcVersion != core.BlockChainVersion && bcVersion != 0 { - return nil, fmt.Errorf("Blockchain DB version mismatch (%d / %d). Run geth upgradedb.\n", bcVersion, core.BlockChainVersion) + return nil, fmt.Errorf("Blockchain DB version mismatch (%d / %d).\n", bcVersion, core.BlockChainVersion) } rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion) } From 355fc47d396298bccf93c37bdbba9b9e88864790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 21 Aug 2018 13:58:10 +0200 Subject: [PATCH 37/96] les: fix CHT field in nodeInfo (#17465) --- les/commons.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/les/commons.go b/les/commons.go index 251b7a5833..d8e9412951 100644 --- a/les/commons.go +++ b/les/commons.go @@ -76,18 +76,30 @@ func (c *lesCommons) makeProtocols(versions []uint) []p2p.Protocol { // nodeInfo retrieves some protocol metadata about the running host node. func (c *lesCommons) nodeInfo() interface{} { var cht light.TrustedCheckpoint - sections, _, sectionHead := c.chtIndexer.Sections() - sections2, _, sectionHead2 := c.bloomTrieIndexer.Sections() + sections, _, _ := c.chtIndexer.Sections() + sections2, _, _ := c.bloomTrieIndexer.Sections() + + if !c.protocolManager.lightSync { + // convert to client section size if running in server mode + sections /= light.CHTFrequencyClient / light.CHTFrequencyServer + } + if sections2 < sections { sections = sections2 - sectionHead = sectionHead2 } if sections > 0 { sectionIndex := sections - 1 + sectionHead := c.bloomTrieIndexer.SectionHead(sectionIndex) + var chtRoot common.Hash + if c.protocolManager.lightSync { + chtRoot = light.GetChtRoot(c.chainDb, sectionIndex, sectionHead) + } else { + chtRoot = light.GetChtV2Root(c.chainDb, sectionIndex, sectionHead) + } cht = light.TrustedCheckpoint{ SectionIdx: sectionIndex, SectionHead: sectionHead, - CHTRoot: light.GetChtRoot(c.chainDb, sectionIndex, sectionHead), + CHTRoot: chtRoot, BloomRoot: light.GetBloomTrieRoot(c.chainDb, sectionIndex, sectionHead), } } From 9f036647e4b3e7c3aa8941dc239f85326a5e5ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 21 Aug 2018 14:39:28 +0300 Subject: [PATCH 38/96] consensus/clique, light: light client snapshots on Rinkeby --- consensus/clique/clique.go | 31 ++++++++++++++-------------- consensus/clique/snapshot_test.go | 2 +- light/lightchain.go | 34 ++++++++++++++++++++----------- light/postprocess.go | 25 ++++++++++++----------- params/config.go | 1 + 5 files changed, 53 insertions(+), 40 deletions(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 59bb3d40b4..0859447014 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -387,22 +387,23 @@ func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash commo break } } - // If we're at block zero, make a snapshot - if number == 0 { - genesis := chain.GetHeaderByNumber(0) - if err := c.VerifyHeader(chain, genesis, false); err != nil { - return nil, err + // If we're at an checkpoint block, make a snapshot if it's known + if number%c.config.Epoch == 0 { + checkpoint := chain.GetHeaderByNumber(number) + if checkpoint != nil { + hash := checkpoint.Hash() + + signers := make([]common.Address, (len(checkpoint.Extra)-extraVanity-extraSeal)/common.AddressLength) + for i := 0; i < len(signers); i++ { + copy(signers[i][:], checkpoint.Extra[extraVanity+i*common.AddressLength:]) + } + snap = newSnapshot(c.config, c.signatures, number, hash, signers) + if err := snap.store(c.db); err != nil { + return nil, err + } + log.Info("Stored checkpoint snapshot to disk", "number", number, "hash", hash) + break } - signers := make([]common.Address, (len(genesis.Extra)-extraVanity-extraSeal)/common.AddressLength) - for i := 0; i < len(signers); i++ { - copy(signers[i][:], genesis.Extra[extraVanity+i*common.AddressLength:]) - } - snap = newSnapshot(c.config, c.signatures, 0, genesis.Hash(), signers) - if err := snap.store(c.db); err != nil { - return nil, err - } - log.Trace("Stored genesis voting snapshot to disk") - break } // No snapshot for this header, gather the header and move backward var header *types.Header diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 5ac730c9e7..17719884f0 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -84,7 +84,7 @@ func (r *testerChainReader) GetHeaderByNumber(number uint64) *types.Header { if number == 0 { return rawdb.ReadHeader(r.db, rawdb.ReadCanonicalHash(r.db, 0), 0) } - panic("not supported") + return nil } // Tests that voting is evaluated correctly for various simple and complex scenarios. diff --git a/light/lightchain.go b/light/lightchain.go index b7e629e88b..b5afe1f0e6 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -464,22 +464,32 @@ func (self *LightChain) GetHeaderByNumberOdr(ctx context.Context, number uint64) func (self *LightChain) Config() *params.ChainConfig { return self.hc.Config() } func (self *LightChain) SyncCht(ctx context.Context) bool { + // If we don't have a CHT indexer, abort if self.odr.ChtIndexer() == nil { return false } - headNum := self.CurrentHeader().Number.Uint64() - chtCount, _, _ := self.odr.ChtIndexer().Sections() - if headNum+1 < chtCount*CHTFrequencyClient { - num := chtCount*CHTFrequencyClient - 1 - header, err := GetHeaderByNumber(ctx, self.odr, num) - if header != nil && err == nil { - self.mu.Lock() - if self.hc.CurrentHeader().Number.Uint64() < header.Number.Uint64() { - self.hc.SetCurrentHeader(header) - } - self.mu.Unlock() - return true + // Ensure the remote CHT head is ahead of us + head := self.CurrentHeader().Number.Uint64() + sections, _, _ := self.odr.ChtIndexer().Sections() + + latest := sections*CHTFrequencyClient - 1 + if clique := self.hc.Config().Clique; clique != nil { + latest -= latest % clique.Epoch // epoch snapshot for clique + } + if head >= latest { + return false + } + // Retrieve the latest useful header and update to it + if header, err := GetHeaderByNumber(ctx, self.odr, latest); header != nil && err == nil { + self.mu.Lock() + defer self.mu.Unlock() + + // Ensure the chain didn't move past the latest block while retrieving it + if self.hc.CurrentHeader().Number.Uint64() < header.Number.Uint64() { + log.Info("Updated latest header based on CHT", "number", header.Number, "hash", header.Hash()) + self.hc.SetCurrentHeader(header) } + return true } return false } diff --git a/light/postprocess.go b/light/postprocess.go index 42985aff05..f105d57b55 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -58,28 +58,29 @@ type TrustedCheckpoint struct { SectionHead, CHTRoot, BloomRoot common.Hash } -var ( - mainnetCheckpoint = TrustedCheckpoint{ +// trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to +var trustedCheckpoints = map[common.Hash]TrustedCheckpoint{ + params.MainnetGenesisHash: { name: "mainnet", SectionIdx: 187, SectionHead: common.HexToHash("e6baa034efa31562d71ff23676512dec6562c1ad0301e08843b907e81958c696"), CHTRoot: common.HexToHash("28001955219719cf06de1b08648969139d123a9835fc760547a1e4dabdabc15a"), BloomRoot: common.HexToHash("395ca2373fc662720ac6b58b3bbe71f68aa0f38b63b2d3553dd32ff3c51eebc4"), - } - - ropstenCheckpoint = TrustedCheckpoint{ + }, + params.TestnetGenesisHash: { name: "ropsten", SectionIdx: 117, SectionHead: common.HexToHash("9529b38631ae30783f56cbe4c3b9f07575b770ecba4f6e20a274b1e2f40fede1"), CHTRoot: common.HexToHash("6f48e9f101f1fac98e7d74fbbcc4fda138358271ffd974d40d2506f0308bb363"), BloomRoot: common.HexToHash("8242342e66e942c0cd893484e6736b9862ceb88b43ca344bb06a8285ac1b6d64"), - } -) - -// trustedCheckpoints associates each known checkpoint with the genesis hash of the chain it belongs to -var trustedCheckpoints = map[common.Hash]TrustedCheckpoint{ - params.MainnetGenesisHash: mainnetCheckpoint, - params.TestnetGenesisHash: ropstenCheckpoint, + }, + params.RinkebyGenesisHash: { + name: "rinkeby", + SectionIdx: 85, + SectionHead: common.HexToHash("92cfa67afc4ad8ab0dcbc6fa49efd14b5b19402442e7317e6bc879d85f89d64d"), + CHTRoot: common.HexToHash("2802ec92cd7a54a75bca96afdc666ae7b99e5d96cf8192dcfb09588812f51564"), + BloomRoot: common.HexToHash("ebefeb31a9a42866d8cf2d2477704b4c3d7c20d0e4e9b5aaa77f396e016a1263"), + }, } var ( diff --git a/params/config.go b/params/config.go index b9e9bb8d6e..70a1edead4 100644 --- a/params/config.go +++ b/params/config.go @@ -27,6 +27,7 @@ import ( var ( MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") TestnetGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") + RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") ) var ( From 86acdf1a5bb31196adbc92e4923e0398868ce876 Mon Sep 17 00:00:00 2001 From: Jeremy Schlatter Date: Tue, 21 Aug 2018 06:13:03 -0700 Subject: [PATCH 39/96] vendor: update rjeczalik/notify so that it compiles on go1.11 (#17467) --- vendor/github.com/rjeczalik/notify/README.md | 1 + .../github.com/rjeczalik/notify/appveyor.yml | 12 +- .../rjeczalik/notify/debug_debug.go | 4 +- .../rjeczalik/notify/debug_nodebug.go | 4 +- .../rjeczalik/notify/watcher_fsevents_cgo.go | 12 +- .../notify/watcher_fsevents_go1.10.go | 14 ++ .../notify/watcher_fsevents_go1.11.go | 9 ++ .../notify/watcher_notimplemented.go | 15 +++ .../rjeczalik/notify/watcher_readdcw.go | 123 +++++++++++------- .../rjeczalik/notify/watcher_stub.go | 22 +--- .../rjeczalik/notify/watcher_trigger.go | 3 +- vendor/vendor.json | 6 +- 12 files changed, 143 insertions(+), 82 deletions(-) create mode 100644 vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go create mode 100644 vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go create mode 100644 vendor/github.com/rjeczalik/notify/watcher_notimplemented.go diff --git a/vendor/github.com/rjeczalik/notify/README.md b/vendor/github.com/rjeczalik/notify/README.md index 02a5f32908..ad743b2a24 100644 --- a/vendor/github.com/rjeczalik/notify/README.md +++ b/vendor/github.com/rjeczalik/notify/README.md @@ -19,3 +19,4 @@ Filesystem event notification library on steroids. (under active development) - [github.com/cortesi/devd](https://github.com/cortesi/devd) - [github.com/cortesi/modd](https://github.com/cortesi/modd) - [github.com/syncthing/syncthing-inotify](https://github.com/syncthing/syncthing-inotify) +- [github.com/OrlovEvgeny/TinyJPG](https://github.com/OrlovEvgeny/TinyJPG) diff --git a/vendor/github.com/rjeczalik/notify/appveyor.yml b/vendor/github.com/rjeczalik/notify/appveyor.yml index a495bd7c73..a0bdc37a38 100644 --- a/vendor/github.com/rjeczalik/notify/appveyor.yml +++ b/vendor/github.com/rjeczalik/notify/appveyor.yml @@ -7,16 +7,20 @@ clone_folder: c:\projects\src\github.com\rjeczalik\notify environment: PATH: c:\projects\bin;%PATH% GOPATH: c:\projects - NOTIFY_TIMEOUT: 5s + NOTIFY_TIMEOUT: 10s + GOVERSION: 1.10.3 install: + - rmdir c:\go /s /q + - appveyor DownloadFile https://storage.googleapis.com/golang/go%GOVERSION%.windows-amd64.zip + - 7z x go%GOVERSION%.windows-amd64.zip -y -oC:\ > NUL + + - cd %APPVEYOR_BUILD_FOLDER% - go version - - go get -v -t ./... build_script: - - go tool vet -all . - go build ./... - - go test -v -timeout 60s -race ./... + - go test -v -timeout 120s -race ./... test: off diff --git a/vendor/github.com/rjeczalik/notify/debug_debug.go b/vendor/github.com/rjeczalik/notify/debug_debug.go index 6fca891ab3..9d234cedda 100644 --- a/vendor/github.com/rjeczalik/notify/debug_debug.go +++ b/vendor/github.com/rjeczalik/notify/debug_debug.go @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Copyright (c) 2014-2018 The Notify Authors. All rights reserved. // Use of this source code is governed by the MIT license that can be // found in the LICENSE file. @@ -6,4 +6,4 @@ package notify -var debugTag bool = true +var debugTag = true diff --git a/vendor/github.com/rjeczalik/notify/debug_nodebug.go b/vendor/github.com/rjeczalik/notify/debug_nodebug.go index be391a2769..9ebf880d88 100644 --- a/vendor/github.com/rjeczalik/notify/debug_nodebug.go +++ b/vendor/github.com/rjeczalik/notify/debug_nodebug.go @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Copyright (c) 2014-2018 The Notify Authors. All rights reserved. // Use of this source code is governed by the MIT license that can be // found in the LICENSE file. @@ -6,4 +6,4 @@ package notify -var debugTag bool = false +var debugTag = false diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go index a2b332a2e0..95ee704444 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go @@ -48,7 +48,7 @@ var wg sync.WaitGroup // used to wait until the runloop starts // started and is ready via the wg. It also serves purpose of a dummy source, // thanks to it the runloop does not return as it also has at least one source // registered. -var source = C.CFRunLoopSourceCreate(nil, 0, &C.CFRunLoopSourceContext{ +var source = C.CFRunLoopSourceCreate(refZero, 0, &C.CFRunLoopSourceContext{ perform: (C.CFRunLoopPerformCallBack)(C.gosource), }) @@ -90,6 +90,10 @@ func gostream(_, info uintptr, n C.size_t, paths, flags, ids uintptr) { if n == 0 { return } + fn := streamFuncs.get(info) + if fn == nil { + return + } ev := make([]FSEvent, 0, int(n)) for i := uintptr(0); i < uintptr(n); i++ { switch flags := *(*uint32)(unsafe.Pointer((flags + i*offflag))); { @@ -104,7 +108,7 @@ func gostream(_, info uintptr, n C.size_t, paths, flags, ids uintptr) { } } - streamFuncs.get(info)(ev) + fn(ev) } // StreamFunc is a callback called when stream receives file events. @@ -162,8 +166,8 @@ func (s *stream) Start() error { return nil } wg.Wait() - p := C.CFStringCreateWithCStringNoCopy(nil, C.CString(s.path), C.kCFStringEncodingUTF8, nil) - path := C.CFArrayCreate(nil, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) + p := C.CFStringCreateWithCStringNoCopy(refZero, C.CString(s.path), C.kCFStringEncodingUTF8, refZero) + path := C.CFArrayCreate(refZero, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) ctx := C.FSEventStreamContext{} ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags) if ref == nilstream { diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go new file mode 100644 index 0000000000..dd2433d20a --- /dev/null +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go @@ -0,0 +1,14 @@ +// Copyright (c) 2018 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,!kqueue,cgo,!go1.11 + +package notify + +/* + #include +*/ +import "C" + +var refZero = (*C.struct___CFAllocator)(nil) diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go new file mode 100644 index 0000000000..92b406ce7c --- /dev/null +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go @@ -0,0 +1,9 @@ +// Copyright (c) 2018 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build darwin,!kqueue,go1.11 + +package notify + +const refZero = 0 diff --git a/vendor/github.com/rjeczalik/notify/watcher_notimplemented.go b/vendor/github.com/rjeczalik/notify/watcher_notimplemented.go new file mode 100644 index 0000000000..bb0672fd88 --- /dev/null +++ b/vendor/github.com/rjeczalik/notify/watcher_notimplemented.go @@ -0,0 +1,15 @@ +// Copyright (c) 2014-2018 The Notify Authors. All rights reserved. +// Use of this source code is governed by the MIT license that can be +// found in the LICENSE file. + +// +build !darwin,!linux,!freebsd,!dragonfly,!netbsd,!openbsd,!windows +// +build !kqueue,!solaris + +package notify + +import "errors" + +// newWatcher stub. +func newWatcher(chan<- EventInfo) watcher { + return watcherStub{errors.New("notify: not implemented")} +} diff --git a/vendor/github.com/rjeczalik/notify/watcher_readdcw.go b/vendor/github.com/rjeczalik/notify/watcher_readdcw.go index 1494fcd799..b69811a690 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_readdcw.go +++ b/vendor/github.com/rjeczalik/notify/watcher_readdcw.go @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Copyright (c) 2014-2018 The Notify Authors. All rights reserved. // Use of this source code is governed by the MIT license that can be // found in the LICENSE file. @@ -22,7 +22,7 @@ import ( const readBufferSize = 4096 // Since all operations which go through the Windows completion routine are done -// asynchronously, filter may set one of the constants belor. They were defined +// asynchronously, filter may set one of the constants below. They were defined // in order to distinguish whether current folder should be re-registered in // ReadDirectoryChangesW function or some control operations need to be executed. const ( @@ -109,8 +109,13 @@ func (g *grip) register(cph syscall.Handle) (err error) { // buffer. Directory changes that occur between calls to this function are added // to the buffer and then, returned with the next call. func (g *grip) readDirChanges() error { + handle := syscall.Handle(atomic.LoadUintptr((*uintptr)(&g.handle))) + if handle == syscall.InvalidHandle { + return nil // Handle was closed. + } + return syscall.ReadDirectoryChanges( - g.handle, + handle, &g.buffer[0], uint32(unsafe.Sizeof(g.buffer)), g.recursive, @@ -220,12 +225,27 @@ func (wd *watched) updateGrip(idx int, cph syscall.Handle, reset bool, // returned from the operating system kernel. func (wd *watched) closeHandle() (err error) { for _, g := range wd.digrip { - if g != nil && g.handle != syscall.InvalidHandle { - switch suberr := syscall.CloseHandle(g.handle); { - case suberr == nil: - g.handle = syscall.InvalidHandle - case err == nil: - err = suberr + if g == nil { + continue + } + + for { + handle := syscall.Handle(atomic.LoadUintptr((*uintptr)(&g.handle))) + if handle == syscall.InvalidHandle { + break // Already closed. + } + + e := syscall.CloseHandle(handle) + if e != nil && err == nil { + err = e + } + + // Set invalid handle even when CloseHandle fails. This will leak + // the handle but, since we can't close it anyway, there won't be + // any difference. + if atomic.CompareAndSwapUintptr((*uintptr)(&g.handle), + (uintptr)(handle), (uintptr)(syscall.InvalidHandle)) { + break } } } @@ -272,50 +292,49 @@ func (r *readdcw) RecursiveWatch(path string, event Event) error { // watch inserts a directory to the group of watched folders. If watched folder // already exists, function tries to rewatch it with new filters(NOT VALID). Moreover, // watch starts the main event loop goroutine when called for the first time. -func (r *readdcw) watch(path string, event Event, recursive bool) (err error) { +func (r *readdcw) watch(path string, event Event, recursive bool) error { if event&^(All|fileNotifyChangeAll) != 0 { return errors.New("notify: unknown event") } + r.Lock() - wd, ok := r.m[path] - r.Unlock() - if !ok { - if err = r.lazyinit(); err != nil { - return - } - r.Lock() - defer r.Unlock() - if wd, ok = r.m[path]; ok { - dbgprint("watch: exists already") - return - } - if wd, err = newWatched(r.cph, uint32(event), recursive, path); err != nil { - return - } - r.m[path] = wd - dbgprint("watch: new watch added") - } else { - dbgprint("watch: exists already") + defer r.Unlock() + + if wd, ok := r.m[path]; ok { + dbgprint("watch: already exists") + wd.filter &^= stateUnwatch + return nil } + + if err := r.lazyinit(); err != nil { + return err + } + + wd, err := newWatched(r.cph, uint32(event), recursive, path) + if err != nil { + return err + } + + r.m[path] = wd + dbgprint("watch: new watch added") + return nil } -// lazyinit creates an I/O completion port and starts the main event processing -// loop. This method uses Double-Checked Locking optimization. +// lazyinit creates an I/O completion port and starts the main event loop. func (r *readdcw) lazyinit() (err error) { invalid := uintptr(syscall.InvalidHandle) + if atomic.LoadUintptr((*uintptr)(&r.cph)) == invalid { - r.Lock() - defer r.Unlock() - if atomic.LoadUintptr((*uintptr)(&r.cph)) == invalid { - cph := syscall.InvalidHandle - if cph, err = syscall.CreateIoCompletionPort(cph, 0, 0, 0); err != nil { - return - } - r.cph, r.start = cph, true - go r.loop() + cph := syscall.InvalidHandle + if cph, err = syscall.CreateIoCompletionPort(cph, 0, 0, 0); err != nil { + return } + + r.cph, r.start = cph, true + go r.loop() } + return } @@ -364,6 +383,7 @@ func (r *readdcw) loopstate(overEx *overlappedEx) { overEx.parent.parent.recreate(r.cph) case stateUnwatch: dbgprint("loopstate unwatch") + overEx.parent.parent.closeHandle() delete(r.m, syscall.UTF16ToString(overEx.parent.pathw)) case stateCPClose: default: @@ -495,27 +515,30 @@ func (r *readdcw) RecursiveUnwatch(path string) error { // TODO : pknap func (r *readdcw) unwatch(path string) (err error) { var wd *watched + r.Lock() defer r.Unlock() if wd, err = r.nonStateWatchedLocked(path); err != nil { return } + wd.filter |= stateUnwatch - if err = wd.closeHandle(); err != nil { - wd.filter &^= stateUnwatch - return - } + dbgprint("unwatch: set unwatch state") + if _, attrErr := syscall.GetFileAttributes(&wd.pathw[0]); attrErr != nil { for _, g := range wd.digrip { - if g != nil { - dbgprint("unwatch: posting") - if err = syscall.PostQueuedCompletionStatus(r.cph, 0, 0, (*syscall.Overlapped)(unsafe.Pointer(g.ovlapped))); err != nil { - wd.filter &^= stateUnwatch - return - } + if g == nil { + continue + } + + dbgprint("unwatch: posting") + if err = syscall.PostQueuedCompletionStatus(r.cph, 0, 0, (*syscall.Overlapped)(unsafe.Pointer(g.ovlapped))); err != nil { + wd.filter &^= stateUnwatch + return } } } + return } diff --git a/vendor/github.com/rjeczalik/notify/watcher_stub.go b/vendor/github.com/rjeczalik/notify/watcher_stub.go index 68b9c135b0..9b284ddc85 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_stub.go +++ b/vendor/github.com/rjeczalik/notify/watcher_stub.go @@ -1,23 +1,13 @@ -// Copyright (c) 2014-2015 The Notify Authors. All rights reserved. +// Copyright (c) 2014-2018 The Notify Authors. All rights reserved. // Use of this source code is governed by the MIT license that can be // found in the LICENSE file. -// +build !darwin,!linux,!freebsd,!dragonfly,!netbsd,!openbsd,!windows -// +build !kqueue,!solaris - package notify -import "errors" - -type stub struct{ error } - -// newWatcher stub. -func newWatcher(chan<- EventInfo) watcher { - return stub{errors.New("notify: not implemented")} -} +type watcherStub struct{ error } // Following methods implement notify.watcher interface. -func (s stub) Watch(string, Event) error { return s } -func (s stub) Rewatch(string, Event, Event) error { return s } -func (s stub) Unwatch(string) (err error) { return s } -func (s stub) Close() error { return s } +func (s watcherStub) Watch(string, Event) error { return s } +func (s watcherStub) Rewatch(string, Event, Event) error { return s } +func (s watcherStub) Unwatch(string) (err error) { return s } +func (s watcherStub) Close() error { return s } diff --git a/vendor/github.com/rjeczalik/notify/watcher_trigger.go b/vendor/github.com/rjeczalik/notify/watcher_trigger.go index 78151f909b..1ebe04829e 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_trigger.go +++ b/vendor/github.com/rjeczalik/notify/watcher_trigger.go @@ -106,7 +106,8 @@ func newWatcher(c chan<- EventInfo) watcher { } t.t = newTrigger(t.pthLkp) if err := t.t.Init(); err != nil { - panic(err) + t.Close() + return watcherStub{fmt.Errorf("failed setting up watcher: %v", err)} } go t.monitor() return t diff --git a/vendor/vendor.json b/vendor/vendor.json index 78886c7bb3..0bfc1ca0c1 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -370,10 +370,10 @@ "revisionTime": "2017-08-14T17:01:13Z" }, { - "checksumSHA1": "28UVHMmHx0iqO0XiJsjx+fwILyI=", + "checksumSHA1": "D8AVDI39CJ+jvw0HOotYU2gz54c=", "path": "github.com/rjeczalik/notify", - "revision": "c31e5f2cb22b3e4ef3f882f413847669bf2652b9", - "revisionTime": "2018-02-03T14:01:15Z" + "revision": "4e54e7fd043e865c50bda93359fb78813a8d165b", + "revisionTime": "2018-08-08T20:39:25Z" }, { "checksumSHA1": "5uqO4ITTDMklKi3uNaE/D9LQ5nM=", From a063fe9b2defdb595068483b4c5df41f1a3a4860 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 21 Aug 2018 22:33:04 +0800 Subject: [PATCH 40/96] miner: fix uncle iteration logic (#17469) --- miner/worker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 2f76f2a92c..23cfaf2256 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -308,7 +308,7 @@ func (w *worker) mainLoop() { return false } uncles = append(uncles, uncle.Header()) - return true + return false }) w.commit(uncles, nil, true, start) } @@ -522,7 +522,7 @@ func (w *worker) updateSnapshot() { return false } uncles = append(uncles, uncle.Header()) - return true + return false }) w.snapshotBlock = types.NewBlock( From 522cfc68ff496aee4205add982db049dc3092024 Mon Sep 17 00:00:00 2001 From: Geon Kim Date: Wed, 22 Aug 2018 03:13:33 +0900 Subject: [PATCH 41/96] swarm: fix typos (#17473) --- swarm/network/simulation/simulation_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/swarm/network/simulation/simulation_test.go b/swarm/network/simulation/simulation_test.go index 8576732c9f..eed09bf508 100644 --- a/swarm/network/simulation/simulation_test.go +++ b/swarm/network/simulation/simulation_test.go @@ -63,7 +63,7 @@ func TestRun(t *testing.T) { } }) - t.Run("cancelation", func(t *testing.T) { + t.Run("cancellation", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() @@ -164,7 +164,7 @@ func TestDone(t *testing.T) { select { case <-time.After(timeout): - t.Error("done channel closing timmed out") + t.Error("done channel closing timed out") case <-sim.Done(): if d := time.Since(start); d < sleep { t.Errorf("done channel closed sooner then expected: %s", d) @@ -172,7 +172,7 @@ func TestDone(t *testing.T) { } } -// a helper map for usual services that do not do anyting +// a helper map for usual services that do not do anything var noopServiceFuncMap = map[string]ServiceFunc{ "noop": noopServiceFunc, } From b2c644ffb5c283a171ddf3889693673939917541 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 22 Aug 2018 03:56:54 +0800 Subject: [PATCH 42/96] cmd, eth, miner: make recommit configurable (#17444) * cmd, eth, miner: make recommit configurable * cmd, eth, les, miner: polish a bit * miner: filter duplicate sealing work * cmd: remove uncessary conversion * miner: avoid microptimization in favor of cleaner code --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 22 ++-- eth/api.go | 6 ++ eth/backend.go | 8 +- eth/config.go | 15 +-- eth/gen_config.go | 53 ++++++++-- internal/web3ext/web3ext.go | 5 + les/backend.go | 2 +- miner/miner.go | 10 +- miner/worker.go | 202 +++++++++++++++++++++++++++++------- miner/worker_test.go | 106 ++++++++++++++++++- 12 files changed, 360 insertions(+), 71 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index a063860513..2e87bb8200 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -106,6 +106,7 @@ var ( utils.MinerLegacyEtherbaseFlag, utils.MinerExtraDataFlag, utils.MinerLegacyExtraDataFlag, + utils.MinerRecommitIntervalFlag, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 9e18f70472..674c5d9015 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -190,6 +190,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerGasTargetFlag, utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, + utils.MinerRecommitIntervalFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e3a8cc2eac..7317655836 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -335,12 +335,12 @@ var ( MinerGasPriceFlag = BigFlag{ Name: "miner.gasprice", Usage: "Minimal gas price for mining a transactions", - Value: eth.DefaultConfig.GasPrice, + Value: eth.DefaultConfig.MinerGasPrice, } MinerLegacyGasPriceFlag = BigFlag{ Name: "gasprice", Usage: "Minimal gas price for mining a transactions (deprecated, use --miner.gasprice)", - Value: eth.DefaultConfig.GasPrice, + Value: eth.DefaultConfig.MinerGasPrice, } MinerEtherbaseFlag = cli.StringFlag{ Name: "miner.etherbase", @@ -360,6 +360,11 @@ var ( Name: "extradata", Usage: "Block extra data set by the miner (default = client version, deprecated, use --miner.extradata)", } + MinerRecommitIntervalFlag = cli.DurationFlag{ + Name: "miner.recommit", + Usage: "Time interval to recreate the block being mined.", + Value: 3 * time.Second, + } // Account settings UnlockedAccountFlag = cli.StringFlag{ Name: "unlock", @@ -1124,16 +1129,19 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.DocRoot = ctx.GlobalString(DocRootFlag.Name) } if ctx.GlobalIsSet(MinerLegacyExtraDataFlag.Name) { - cfg.ExtraData = []byte(ctx.GlobalString(MinerLegacyExtraDataFlag.Name)) + cfg.MinerExtraData = []byte(ctx.GlobalString(MinerLegacyExtraDataFlag.Name)) } if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { - cfg.ExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) + cfg.MinerExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) } if ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { - cfg.GasPrice = GlobalBig(ctx, MinerLegacyGasPriceFlag.Name) + cfg.MinerGasPrice = GlobalBig(ctx, MinerLegacyGasPriceFlag.Name) } if ctx.GlobalIsSet(MinerGasPriceFlag.Name) { - cfg.GasPrice = GlobalBig(ctx, MinerGasPriceFlag.Name) + cfg.MinerGasPrice = GlobalBig(ctx, MinerGasPriceFlag.Name) + } + if ctx.GlobalIsSet(MinerRecommitIntervalFlag.Name) { + cfg.MinerRecommit = ctx.Duration(MinerRecommitIntervalFlag.Name) } if ctx.GlobalIsSet(VMEnableDebugFlag.Name) { // TODO(fjl): force-enable this in --dev mode @@ -1176,7 +1184,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.GlobalInt(DeveloperPeriodFlag.Name)), developer.Address) if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) && !ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { - cfg.GasPrice = big.NewInt(1) + cfg.MinerGasPrice = big.NewInt(1) } } // TODO(fjl): move trie cache generations into config diff --git a/eth/api.go b/eth/api.go index c1fbcb6d40..4b0ba8edbd 100644 --- a/eth/api.go +++ b/eth/api.go @@ -25,6 +25,7 @@ import ( "math/big" "os" "strings" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -160,6 +161,11 @@ func (api *PrivateMinerAPI) SetEtherbase(etherbase common.Address) bool { return true } +// SetRecommitInterval updates the interval for miner sealing work recommitting. +func (api *PrivateMinerAPI) SetRecommitInterval(interval int) { + api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond) +} + // GetHashrate returns the current hashrate of the miner. func (api *PrivateMinerAPI) GetHashrate() uint64 { return api.e.miner.HashRate() diff --git a/eth/backend.go b/eth/backend.go index 588b782568..648175acfd 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -127,7 +127,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { engine: CreateConsensusEngine(ctx, chainConfig, &config.Ethash, config.MinerNotify, chainDb), shutdownChan: make(chan bool), networkID: config.NetworkId, - gasPrice: config.GasPrice, + gasPrice: config.MinerGasPrice, etherbase: config.Etherbase, bloomRequests: make(chan chan *bloombits.Retrieval), bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, bloomConfirms), @@ -167,13 +167,13 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine) - eth.miner.SetExtra(makeExtraData(config.ExtraData)) + eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine, config.MinerRecommit) + eth.miner.SetExtra(makeExtraData(config.MinerExtraData)) eth.APIBackend = &EthAPIBackend{eth, nil} gpoParams := config.GPO if gpoParams.Default == nil { - gpoParams.Default = config.GasPrice + gpoParams.Default = config.MinerGasPrice } eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) diff --git a/eth/config.go b/eth/config.go index 0c82f29232..cbd02416be 100644 --- a/eth/config.go +++ b/eth/config.go @@ -48,7 +48,7 @@ var DefaultConfig = Config{ DatabaseCache: 768, TrieCache: 256, TrieTimeout: 60 * time.Minute, - GasPrice: big.NewInt(18 * params.Shannon), + MinerGasPrice: big.NewInt(18 * params.Shannon), TxPool: core.DefaultTxPoolConfig, GPO: gasprice.Config{ @@ -95,11 +95,12 @@ type Config struct { TrieTimeout time.Duration // Mining-related options - Etherbase common.Address `toml:",omitempty"` - MinerThreads int `toml:",omitempty"` - MinerNotify []string `toml:",omitempty"` - ExtraData []byte `toml:",omitempty"` - GasPrice *big.Int + Etherbase common.Address `toml:",omitempty"` + MinerThreads int `toml:",omitempty"` + MinerNotify []string `toml:",omitempty"` + MinerExtraData []byte `toml:",omitempty"` + MinerGasPrice *big.Int + MinerRecommit time.Duration // Ethash options Ethash ethash.Config @@ -118,5 +119,5 @@ type Config struct { } type configMarshaling struct { - ExtraData hexutil.Bytes + MinerExtraData hexutil.Bytes } diff --git a/eth/gen_config.go b/eth/gen_config.go index 4f2e82d941..62556be7e1 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -4,6 +4,7 @@ package eth import ( "math/big" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -15,20 +16,26 @@ import ( var _ = (*configMarshaling)(nil) +// MarshalTOML marshals as TOML. func (c Config) MarshalTOML() (interface{}, error) { type Config struct { Genesis *core.Genesis `toml:",omitempty"` NetworkId uint64 SyncMode downloader.SyncMode + NoPruning bool LightServ int `toml:",omitempty"` LightPeers int `toml:",omitempty"` SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int + TrieCache int + TrieTimeout time.Duration Etherbase common.Address `toml:",omitempty"` MinerThreads int `toml:",omitempty"` - ExtraData hexutil.Bytes `toml:",omitempty"` - GasPrice *big.Int + MinerNotify []string `toml:",omitempty"` + MinerExtraData hexutil.Bytes `toml:",omitempty"` + MinerGasPrice *big.Int + MinerRecommit time.Duration Ethash ethash.Config TxPool core.TxPoolConfig GPO gasprice.Config @@ -39,15 +46,20 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.Genesis = c.Genesis enc.NetworkId = c.NetworkId enc.SyncMode = c.SyncMode + enc.NoPruning = c.NoPruning enc.LightServ = c.LightServ enc.LightPeers = c.LightPeers enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache + enc.TrieCache = c.TrieCache + enc.TrieTimeout = c.TrieTimeout enc.Etherbase = c.Etherbase enc.MinerThreads = c.MinerThreads - enc.ExtraData = c.ExtraData - enc.GasPrice = c.GasPrice + enc.MinerNotify = c.MinerNotify + enc.MinerExtraData = c.MinerExtraData + enc.MinerGasPrice = c.MinerGasPrice + enc.MinerRecommit = c.MinerRecommit enc.Ethash = c.Ethash enc.TxPool = c.TxPool enc.GPO = c.GPO @@ -56,20 +68,26 @@ func (c Config) MarshalTOML() (interface{}, error) { return &enc, nil } +// UnmarshalTOML unmarshals from TOML. func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { type Config struct { Genesis *core.Genesis `toml:",omitempty"` NetworkId *uint64 SyncMode *downloader.SyncMode + NoPruning *bool LightServ *int `toml:",omitempty"` LightPeers *int `toml:",omitempty"` SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int + TrieCache *int + TrieTimeout *time.Duration Etherbase *common.Address `toml:",omitempty"` MinerThreads *int `toml:",omitempty"` - ExtraData *hexutil.Bytes `toml:",omitempty"` - GasPrice *big.Int + MinerNotify []string `toml:",omitempty"` + MinerExtraData *hexutil.Bytes `toml:",omitempty"` + MinerGasPrice *big.Int + MinerRecommit *time.Duration Ethash *ethash.Config TxPool *core.TxPoolConfig GPO *gasprice.Config @@ -89,6 +107,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SyncMode != nil { c.SyncMode = *dec.SyncMode } + if dec.NoPruning != nil { + c.NoPruning = *dec.NoPruning + } if dec.LightServ != nil { c.LightServ = *dec.LightServ } @@ -104,17 +125,29 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.DatabaseCache != nil { c.DatabaseCache = *dec.DatabaseCache } + if dec.TrieCache != nil { + c.TrieCache = *dec.TrieCache + } + if dec.TrieTimeout != nil { + c.TrieTimeout = *dec.TrieTimeout + } if dec.Etherbase != nil { c.Etherbase = *dec.Etherbase } if dec.MinerThreads != nil { c.MinerThreads = *dec.MinerThreads } - if dec.ExtraData != nil { - c.ExtraData = *dec.ExtraData + if dec.MinerNotify != nil { + c.MinerNotify = dec.MinerNotify } - if dec.GasPrice != nil { - c.GasPrice = dec.GasPrice + if dec.MinerExtraData != nil { + c.MinerExtraData = *dec.MinerExtraData + } + if dec.MinerGasPrice != nil { + c.MinerGasPrice = dec.MinerGasPrice + } + if dec.MinerRecommit != nil { + c.MinerRecommit = *dec.MinerRecommit } if dec.Ethash != nil { c.Ethash = *dec.Ethash diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 000e3728de..f4eb47a12a 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -519,6 +519,11 @@ web3._extend({ params: 1, inputFormatter: [web3._extend.utils.fromDecimal] }), + new web3._extend.Method({ + name: 'setRecommitInterval', + call: 'miner_setRecommitInterval', + params: 1, + }), new web3._extend.Method({ name: 'getHashrate', call: 'miner_getHashrate' diff --git a/les/backend.go b/les/backend.go index d26c1470fe..00025ba634 100644 --- a/les/backend.go +++ b/les/backend.go @@ -141,7 +141,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { leth.ApiBackend = &LesApiBackend{leth, nil} gpoParams := config.GPO if gpoParams.Default == nil { - gpoParams.Default = config.GasPrice + gpoParams.Default = config.MinerGasPrice } leth.ApiBackend.gpo = gasprice.NewOracle(leth.ApiBackend, gpoParams) return leth, nil diff --git a/miner/miner.go b/miner/miner.go index e350e456e9..c5a0c9d62a 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -20,6 +20,7 @@ package miner import ( "fmt" "sync/atomic" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" @@ -51,13 +52,13 @@ type Miner struct { shouldStart int32 // should start indicates whether we should start after sync } -func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine) *Miner { +func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, recommit time.Duration) *Miner { miner := &Miner{ eth: eth, mux: mux, engine: engine, exitCh: make(chan struct{}), - worker: newWorker(config, engine, eth, mux), + worker: newWorker(config, engine, eth, mux, recommit), canStart: 1, } go miner.update() @@ -144,6 +145,11 @@ func (self *Miner) SetExtra(extra []byte) error { return nil } +// SetRecommitInterval sets the interval for sealing work resubmitting. +func (self *Miner) SetRecommitInterval(interval time.Duration) { + self.worker.setRecommitInterval(interval) +} + // Pending returns the currently pending block and associated state. func (self *Miner) Pending() (*types.Block, *state.StateDB) { return self.worker.pending() diff --git a/miner/worker.go b/miner/worker.go index 23cfaf2256..c299ff9dc1 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -51,12 +51,27 @@ const ( // chainSideChanSize is the size of channel listening to ChainSideEvent. chainSideChanSize = 10 + // resubmitAdjustChanSize is the size of resubmitting interval adjustment channel. + resubmitAdjustChanSize = 10 + // miningLogAtDepth is the number of confirmations before logging successful mining. miningLogAtDepth = 5 - // blockRecommitInterval is the time interval to recreate the mining block with + // minRecommitInterval is the minimal time interval to recreate the mining block with // any newly arrived transactions. - blockRecommitInterval = 3 * time.Second + minRecommitInterval = 1 * time.Second + + // maxRecommitInterval is the maximum time interval to recreate the mining block with + // any newly arrived transactions. + maxRecommitInterval = 15 * time.Second + + // intervalAdjustRatio is the impact a single interval adjustment has on sealing work + // resubmitting interval. + intervalAdjustRatio = 0.1 + + // intervalAdjustBias is applied during the new resubmit interval calculation in favor of + // increasing upper limit or decreasing lower limit so that the limit can be reachable. + intervalAdjustBias = 200 * 1000.0 * 1000.0 ) // environment is the worker's current environment and holds all of the current state information. @@ -89,11 +104,18 @@ const ( commitInterruptResubmit ) +// newWorkReq represents a request for new sealing work submitting with relative interrupt notifier. type newWorkReq struct { interrupt *int32 noempty bool } +// intervalAdjust represents a resubmitting interval adjustment. +type intervalAdjust struct { + ratio float64 + inc bool +} + // worker is the main object which takes care of submitting new work to consensus engine // and gathering the sealing result. type worker struct { @@ -112,11 +134,13 @@ type worker struct { chainSideSub event.Subscription // Channels - newWorkCh chan *newWorkReq - taskCh chan *task - resultCh chan *task - startCh chan struct{} - exitCh chan struct{} + newWorkCh chan *newWorkReq + taskCh chan *task + resultCh chan *task + startCh chan struct{} + exitCh chan struct{} + resubmitIntervalCh chan time.Duration + resubmitAdjustCh chan *intervalAdjust current *environment // An environment for current running cycle. possibleUncles map[common.Hash]*types.Block // A set of side blocks as the possible uncle blocks. @@ -132,30 +156,34 @@ type worker struct { // atomic status counters running int32 // The indicator whether the consensus engine is running or not. + newTxs int32 // New arrival transaction count since last sealing work submitting. // Test hooks - newTaskHook func(*task) // Method to call upon receiving a new sealing task - skipSealHook func(*task) bool // Method to decide whether skipping the sealing. - fullTaskHook func() // Method to call before pushing the full sealing task + newTaskHook func(*task) // Method to call upon receiving a new sealing task. + skipSealHook func(*task) bool // Method to decide whether skipping the sealing. + fullTaskHook func() // Method to call before pushing the full sealing task. + resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. } -func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux) *worker { +func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, recommit time.Duration) *worker { worker := &worker{ - config: config, - engine: engine, - eth: eth, - mux: mux, - chain: eth.BlockChain(), - possibleUncles: make(map[common.Hash]*types.Block), - unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), - txsCh: make(chan core.NewTxsEvent, txChanSize), - chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), - chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), - newWorkCh: make(chan *newWorkReq), - taskCh: make(chan *task), - resultCh: make(chan *task, resultQueueSize), - exitCh: make(chan struct{}), - startCh: make(chan struct{}, 1), + config: config, + engine: engine, + eth: eth, + mux: mux, + chain: eth.BlockChain(), + possibleUncles: make(map[common.Hash]*types.Block), + unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), + txsCh: make(chan core.NewTxsEvent, txChanSize), + chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), + chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), + newWorkCh: make(chan *newWorkReq), + taskCh: make(chan *task), + resultCh: make(chan *task, resultQueueSize), + exitCh: make(chan struct{}), + startCh: make(chan struct{}, 1), + resubmitIntervalCh: make(chan time.Duration), + resubmitAdjustCh: make(chan *intervalAdjust, resubmitAdjustChanSize), } // Subscribe NewTxsEvent for tx pool worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) @@ -163,8 +191,14 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, worker.chainHeadSub = eth.BlockChain().SubscribeChainHeadEvent(worker.chainHeadCh) worker.chainSideSub = eth.BlockChain().SubscribeChainSideEvent(worker.chainSideCh) + // Sanitize recommit interval if the user-specified one is too short. + if recommit < minRecommitInterval { + log.Warn("Sanitizing miner recommit interval", "provided", recommit, "updated", minRecommitInterval) + recommit = minRecommitInterval + } + go worker.mainLoop() - go worker.newWorkLoop() + go worker.newWorkLoop(recommit) go worker.resultLoop() go worker.taskLoop() @@ -188,6 +222,11 @@ func (w *worker) setExtra(extra []byte) { w.extra = extra } +// setRecommitInterval updates the interval for miner sealing work recommitting. +func (w *worker) setRecommitInterval(interval time.Duration) { + w.resubmitIntervalCh <- interval +} + // pending returns the pending state and corresponding block. func (w *worker) pending() (*types.Block, *state.StateDB) { // return a snapshot to avoid contention on currentMu mutex @@ -238,35 +277,94 @@ func (w *worker) close() { } // newWorkLoop is a standalone goroutine to submit new mining work upon received events. -func (w *worker) newWorkLoop() { - var interrupt *int32 +func (w *worker) newWorkLoop(recommit time.Duration) { + var ( + interrupt *int32 + minRecommit = recommit // minimal resubmit interval specified by user. + ) timer := time.NewTimer(0) <-timer.C // discard the initial tick - // recommit aborts in-flight transaction execution with given signal and resubmits a new one. - recommit := func(noempty bool, s int32) { + // commit aborts in-flight transaction execution with given signal and resubmits a new one. + commit := func(noempty bool, s int32) { if interrupt != nil { atomic.StoreInt32(interrupt, s) } interrupt = new(int32) w.newWorkCh <- &newWorkReq{interrupt: interrupt, noempty: noempty} - timer.Reset(blockRecommitInterval) + timer.Reset(recommit) + atomic.StoreInt32(&w.newTxs, 0) + } + // recalcRecommit recalculates the resubmitting interval upon feedback. + recalcRecommit := func(target float64, inc bool) { + var ( + prev = float64(recommit.Nanoseconds()) + next float64 + ) + if inc { + next = prev*(1-intervalAdjustRatio) + intervalAdjustRatio*(target+intervalAdjustBias) + // Recap if interval is larger than the maximum time interval + if next > float64(maxRecommitInterval.Nanoseconds()) { + next = float64(maxRecommitInterval.Nanoseconds()) + } + } else { + next = prev*(1-intervalAdjustRatio) + intervalAdjustRatio*(target-intervalAdjustBias) + // Recap if interval is less than the user specified minimum + if next < float64(minRecommit.Nanoseconds()) { + next = float64(minRecommit.Nanoseconds()) + } + } + recommit = time.Duration(int64(next)) } for { select { case <-w.startCh: - recommit(false, commitInterruptNewHead) + commit(false, commitInterruptNewHead) case <-w.chainHeadCh: - recommit(false, commitInterruptNewHead) + commit(false, commitInterruptNewHead) case <-timer.C: // If mining is running resubmit a new work cycle periodically to pull in // higher priced transactions. Disable this overhead for pending blocks. if w.isRunning() && (w.config.Clique == nil || w.config.Clique.Period > 0) { - recommit(true, commitInterruptResubmit) + // Short circuit if no new transaction arrives. + if atomic.LoadInt32(&w.newTxs) == 0 { + timer.Reset(recommit) + continue + } + commit(true, commitInterruptResubmit) + } + + case interval := <-w.resubmitIntervalCh: + // Adjust resubmit interval explicitly by user. + if interval < minRecommitInterval { + log.Warn("Sanitizing miner recommit interval", "provided", interval, "updated", minRecommitInterval) + interval = minRecommitInterval + } + log.Info("Miner recommit interval update", "from", minRecommit, "to", interval) + minRecommit, recommit = interval, interval + + if w.resubmitHook != nil { + w.resubmitHook(minRecommit, recommit) + } + + case adjust := <-w.resubmitAdjustCh: + // Adjust resubmit interval by feedback. + if adjust.inc { + before := recommit + recalcRecommit(float64(recommit.Nanoseconds())/adjust.ratio, true) + log.Trace("Increase miner recommit interval", "from", before, "to", recommit) + } else { + before := recommit + recalcRecommit(float64(minRecommit.Nanoseconds()), false) + log.Trace("Decrease miner recommit interval", "from", before, "to", recommit) + } + + if w.resubmitHook != nil { + w.resubmitHook(minRecommit, recommit) } case <-w.exitCh: @@ -339,6 +437,7 @@ func (w *worker) mainLoop() { w.commitNewWork(nil, false) } } + atomic.AddInt32(&w.newTxs, int32(len(ev.Txs))) // System stopped case <-w.exitCh: @@ -383,7 +482,10 @@ func (w *worker) seal(t *task, stop <-chan struct{}) { // taskLoop is a standalone goroutine to fetch sealing task from the generator and // push them to consensus engine. func (w *worker) taskLoop() { - var stopCh chan struct{} + var ( + stopCh chan struct{} + prev common.Hash + ) // interrupt aborts the in-flight sealing task. interrupt := func() { @@ -398,8 +500,13 @@ func (w *worker) taskLoop() { if w.newTaskHook != nil { w.newTaskHook(task) } + // Reject duplicate sealing work due to resubmitting. + if task.block.HashNoNonce() == prev { + continue + } interrupt() stopCh = make(chan struct{}) + prev = task.block.HashNoNonce() go w.seal(task, stopCh) case <-w.exitCh: interrupt() @@ -414,11 +521,15 @@ func (w *worker) resultLoop() { for { select { case result := <-w.resultCh: + // Short circuit when receiving empty result. if result == nil { continue } + // Short circuit when receiving duplicate result caused by resubmitting. block := result.block - + if w.chain.HasBlock(block.Hash(), block.NumberU64()) { + continue + } // Update the block hash in all logs since it is now available and not when the // receipt/log of individual transactions were created. for _, r := range result.receipts { @@ -568,8 +679,18 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin // (3) worker recreate the mining block with any newly arrived transactions, the interrupt signal is 2. // For the first two cases, the semi-finished work will be discarded. // For the third case, the semi-finished work will be submitted to the consensus engine. - // TODO(rjl493456442) give feedback to newWorkLoop to adjust resubmit interval if it is too short. if interrupt != nil && atomic.LoadInt32(interrupt) != commitInterruptNone { + // Notify resubmit loop to increase resubmitting interval due to too frequent commits. + if atomic.LoadInt32(interrupt) == commitInterruptResubmit { + ratio := float64(w.current.header.GasLimit-w.current.gasPool.Gas()) / float64(w.current.header.GasLimit) + if ratio < 0.1 { + ratio = 0.1 + } + w.resubmitAdjustCh <- &intervalAdjust{ + ratio: ratio, + inc: true, + } + } return atomic.LoadInt32(interrupt) == commitInterruptNewHead } // If we don't have enough gas for any further transactions then we're done @@ -644,6 +765,11 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin } go w.mux.Post(core.PendingLogsEvent{Logs: cpy}) } + // Notify resubmit loop to decrease resubmitting interval if current interval is larger + // than the user-specified one. + if interrupt != nil { + w.resubmitAdjustCh <- &intervalAdjust{inc: false} + } return false } diff --git a/miner/worker_test.go b/miner/worker_test.go index 34bb7f5f33..16708c18c6 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -119,7 +119,7 @@ func (b *testWorkerBackend) PostChainEvents(events []interface{}) { func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) (*worker, *testWorkerBackend) { backend := newTestWorkerBackend(t, chainConfig, engine) backend.txPool.AddLocals(pendingTxs) - w := newWorker(chainConfig, engine, backend, new(event.TypeMux)) + w := newWorker(chainConfig, engine, backend, new(event.TypeMux), time.Second) w.setEtherbase(testBankAddress) return w, backend } @@ -327,7 +327,7 @@ func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, en } } b.txPool.AddLocals(newTxs) - time.Sleep(3 * time.Second) + time.Sleep(time.Second) select { case <-taskCh: @@ -335,3 +335,105 @@ func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, en t.Error("new task timeout") } } + +func TestAdjustIntervalEthash(t *testing.T) { + testAdjustInterval(t, ethashChainConfig, ethash.NewFaker()) +} + +func TestAdjustIntervalClique(t *testing.T) { + testAdjustInterval(t, cliqueChainConfig, clique.New(cliqueChainConfig.Clique, ethdb.NewMemDatabase())) +} + +func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { + defer engine.Close() + + w, _ := newTestWorker(t, chainConfig, engine) + defer w.close() + + w.skipSealHook = func(task *task) bool { + return true + } + w.fullTaskHook = func() { + time.Sleep(100 * time.Millisecond) + } + var ( + progress = make(chan struct{}, 10) + result = make([]float64, 0, 10) + index = 0 + start = false + ) + w.resubmitHook = func(minInterval time.Duration, recommitInterval time.Duration) { + // Short circuit if interval checking hasn't started. + if !start { + return + } + var wantMinInterval, wantRecommitInterval time.Duration + + switch index { + case 0: + wantMinInterval, wantRecommitInterval = 3*time.Second, 3*time.Second + case 1: + origin := float64(3 * time.Second.Nanoseconds()) + estimate := origin*(1-intervalAdjustRatio) + intervalAdjustRatio*(origin/0.8+intervalAdjustBias) + wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(int(estimate))*time.Nanosecond + case 2: + estimate := result[index-1] + min := float64(3 * time.Second.Nanoseconds()) + estimate = estimate*(1-intervalAdjustRatio) + intervalAdjustRatio*(min-intervalAdjustBias) + wantMinInterval, wantRecommitInterval = 3*time.Second, time.Duration(int(estimate))*time.Nanosecond + case 3: + wantMinInterval, wantRecommitInterval = time.Second, time.Second + } + + // Check interval + if minInterval != wantMinInterval { + t.Errorf("resubmit min interval mismatch want %s has %s", wantMinInterval, minInterval) + } + if recommitInterval != wantRecommitInterval { + t.Errorf("resubmit interval mismatch want %s has %s", wantRecommitInterval, recommitInterval) + } + result = append(result, float64(recommitInterval.Nanoseconds())) + index += 1 + progress <- struct{}{} + } + // Ensure worker has finished initialization + for { + b := w.pendingBlock() + if b != nil && b.NumberU64() == 1 { + break + } + } + + w.start() + + time.Sleep(time.Second) + + start = true + w.setRecommitInterval(3 * time.Second) + select { + case <-progress: + case <-time.NewTimer(time.Second).C: + t.Error("interval reset timeout") + } + + w.resubmitAdjustCh <- &intervalAdjust{inc: true, ratio: 0.8} + select { + case <-progress: + case <-time.NewTimer(time.Second).C: + t.Error("interval reset timeout") + } + + w.resubmitAdjustCh <- &intervalAdjust{inc: false} + select { + case <-progress: + case <-time.NewTimer(time.Second).C: + t.Error("interval reset timeout") + } + + w.setRecommitInterval(500 * time.Millisecond) + select { + case <-progress: + case <-time.NewTimer(time.Second).C: + t.Error("interval reset timeout") + } +} From e0d0e64ce22111a2d5492fe6f6d6a0023477e51f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 21 Aug 2018 20:30:06 +0300 Subject: [PATCH 43/96] cmd, core, miner: add --txpool.locals and priority mining --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 14 ++++++++++++++ core/tx_pool.go | 39 +++++++++++++++++++++++++++++++++++---- miner/worker.go | 23 +++++++++++++++++++---- 5 files changed, 70 insertions(+), 8 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 2e87bb8200..4b86382bdb 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -72,6 +72,7 @@ var ( utils.EthashDatasetDirFlag, utils.EthashDatasetsInMemoryFlag, utils.EthashDatasetsOnDiskFlag, + utils.TxPoolLocalsFlag, utils.TxPoolNoLocalsFlag, utils.TxPoolJournalFlag, utils.TxPoolRejournalFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 674c5d9015..1e27d0ae8d 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -114,6 +114,7 @@ var AppHelpFlagGroups = []flagGroup{ { Name: "TRANSACTION POOL", Flags: []cli.Flag{ + utils.TxPoolLocalsFlag, utils.TxPoolNoLocalsFlag, utils.TxPoolJournalFlag, utils.TxPoolRejournalFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7317655836..cfca7b4ab4 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -233,6 +233,10 @@ var ( Value: eth.DefaultConfig.Ethash.DatasetsOnDisk, } // Transaction pool settings + TxPoolLocalsFlag = cli.StringFlag{ + Name: "txpool.locals", + Usage: "Comma separated accounts to treat as locals (no flush, priority inclusion)", + } TxPoolNoLocalsFlag = cli.BoolFlag{ Name: "txpool.nolocals", Usage: "Disables price exemptions for locally submitted transactions", @@ -977,6 +981,16 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config) { } func setTxPool(ctx *cli.Context, cfg *core.TxPoolConfig) { + if ctx.GlobalIsSet(TxPoolLocalsFlag.Name) { + locals := strings.Split(ctx.GlobalString(TxPoolLocalsFlag.Name), ",") + for _, account := range locals { + if trimmed := strings.TrimSpace(account); !common.IsHexAddress(trimmed) { + Fatalf("Invalid account in --txpool.locals: %s", trimmed) + } else { + cfg.Locals = append(cfg.Locals, common.HexToAddress(account)) + } + } + } if ctx.GlobalIsSet(TxPoolNoLocalsFlag.Name) { cfg.NoLocals = ctx.GlobalBool(TxPoolNoLocalsFlag.Name) } diff --git a/core/tx_pool.go b/core/tx_pool.go index 7007f85ddf..46ae2759bc 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -123,9 +123,10 @@ type blockChain interface { // TxPoolConfig are the configuration parameters of the transaction pool. type TxPoolConfig struct { - NoLocals bool // Whether local transaction handling should be disabled - Journal string // Journal of local transactions to survive node restarts - Rejournal time.Duration // Time interval to regenerate the local transaction journal + Locals []common.Address // Addresses that should be treated by default as local + NoLocals bool // Whether local transaction handling should be disabled + Journal string // Journal of local transactions to survive node restarts + Rejournal time.Duration // Time interval to regenerate the local transaction journal PriceLimit uint64 // Minimum gas price to enforce for acceptance into the pool PriceBump uint64 // Minimum price bump percentage to replace an already existing transaction (nonce) @@ -231,6 +232,10 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block gasPrice: new(big.Int).SetUint64(config.PriceLimit), } pool.locals = newAccountSet(pool.signer) + for _, addr := range config.Locals { + log.Info("Setting new local account", "address", addr) + pool.locals.add(addr) + } pool.priced = newTxPricedList(pool.all) pool.reset(nil, chain.CurrentBlock().Header()) @@ -534,6 +539,14 @@ func (pool *TxPool) Pending() (map[common.Address]types.Transactions, error) { return pending, nil } +// Locals retrieves the accounts currently considered local by the pool. +func (pool *TxPool) Locals() []common.Address { + pool.mu.Lock() + defer pool.mu.Unlock() + + return pool.locals.flatten() +} + // local retrieves all currently known local transactions, groupped by origin // account and sorted by nonce. The returned transaction set is a copy and can be // freely modified by calling code. @@ -665,7 +678,10 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (bool, error) { } // Mark local addresses and journal local transactions if local { - pool.locals.add(from) + if !pool.locals.contains(from) { + log.Info("Setting new local account", "address", from) + pool.locals.add(from) + } } pool.journalTx(from, tx) @@ -1138,6 +1154,7 @@ func (a addressesByHeartbeat) Swap(i, j int) { a[i], a[j] = a[j], a[i] } type accountSet struct { accounts map[common.Address]struct{} signer types.Signer + cache *[]common.Address } // newAccountSet creates a new address set with an associated signer for sender @@ -1167,6 +1184,20 @@ func (as *accountSet) containsTx(tx *types.Transaction) bool { // add inserts a new address into the set to track. func (as *accountSet) add(addr common.Address) { as.accounts[addr] = struct{}{} + as.cache = nil +} + +// flatten returns the list of addresses within this set, also caching it for later +// reuse. The returned slice should not be changed! +func (as *accountSet) flatten() []common.Address { + if as.cache == nil { + accounts := make([]common.Address, 0, len(as.accounts)) + for account := range as.accounts { + accounts = append(accounts, account) + } + as.cache = &accounts + } + return *as.cache } // txLookup is used internally by TxPool to track transactions while allowing lookup without diff --git a/miner/worker.go b/miner/worker.go index c299ff9dc1..8c3337ba45 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -877,11 +877,26 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool) { w.updateSnapshot() return } - txs := types.NewTransactionsByPriceAndNonce(w.current.signer, pending) - if w.commitTransactions(txs, w.coinbase, interrupt) { - return + // Split the pending transactions into locals and remotes + localTxs, remoteTxs := make(map[common.Address]types.Transactions), pending + for _, account := range w.eth.TxPool().Locals() { + if txs := remoteTxs[account]; len(txs) > 0 { + delete(remoteTxs, account) + localTxs[account] = txs + } + } + if len(localTxs) > 0 { + txs := types.NewTransactionsByPriceAndNonce(w.current.signer, localTxs) + if w.commitTransactions(txs, w.coinbase, interrupt) { + return + } + } + if len(remoteTxs) > 0 { + txs := types.NewTransactionsByPriceAndNonce(w.current.signer, remoteTxs) + if w.commitTransactions(txs, w.coinbase, interrupt) { + return + } } - w.commit(uncles, w.fullTaskHook, true, tstart) } From af85d8e2b342aa6feff68cec59db9be3c1e85022 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 22 Aug 2018 14:47:50 +0800 Subject: [PATCH 44/96] cmd, eth: apply default miner recommit setting (#17479) --- cmd/utils/flags.go | 2 +- eth/config.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index cfca7b4ab4..c9a936dd5f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -367,7 +367,7 @@ var ( MinerRecommitIntervalFlag = cli.DurationFlag{ Name: "miner.recommit", Usage: "Time interval to recreate the block being mined.", - Value: 3 * time.Second, + Value: eth.DefaultConfig.MinerRecommit, } // Account settings UnlockedAccountFlag = cli.StringFlag{ diff --git a/eth/config.go b/eth/config.go index cbd02416be..1398a55145 100644 --- a/eth/config.go +++ b/eth/config.go @@ -49,6 +49,7 @@ var DefaultConfig = Config{ TrieCache: 256, TrieTimeout: 60 * time.Minute, MinerGasPrice: big.NewInt(18 * params.Shannon), + MinerRecommit: 3 * time.Second, TxPool: core.DefaultTxPoolConfig, GPO: gasprice.Config{ From 316fc7ecfc10d06603f1358c1f4c1020ec36dd2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 22 Aug 2018 11:39:00 +0300 Subject: [PATCH 45/96] params, swarm: release Geth v1.8.14 and Swarm v0.3.2 --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 44c148e9c3..9cb027bd55 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 14 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 14 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index bb9935e24a..1f0c646197 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 2 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 2993fb519d7bb70163765c33bc95c5260ecb003a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 22 Aug 2018 11:42:03 +0300 Subject: [PATCH 46/96] params, swarm: begin geth v1.8.15 and swarm v0.3.3 cycle --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 9cb027bd55..f65236a85c 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 14 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 15 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 1f0c646197..5672e19f81 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 2 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 3 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From f34f361ca6635690f6dd81d6f3bddfff498e9fd6 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 22 Aug 2018 16:53:33 +0200 Subject: [PATCH 47/96] swarm/api/http: fixed resolver bug (#17483) --- swarm/api/http/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index b5ea0c23d4..2aa1963969 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -820,7 +820,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) { manifestAddr := uri.Address() if manifestAddr == nil { - manifestAddr, err = s.api.ResolveURI(r.Context(), uri, credentials) + manifestAddr, err = s.api.Resolve(r.Context(), uri.Addr) if err != nil { getFileFail.Inc(1) RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound) From 1df1187d831a1c62f61e8c90901dc78b5f86943a Mon Sep 17 00:00:00 2001 From: Mymskmkt <1847234666@qq.com> Date: Thu, 23 Aug 2018 16:47:43 +0800 Subject: [PATCH 48/96] p2p: fix comment typo (#17491) --- p2p/dial.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/dial.go b/p2p/dial.go index d8feceb9f3..4baca7d931 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -146,7 +146,7 @@ func newDialState(static []*discover.Node, bootnodes []*discover.Node, ntab disc } func (s *dialstate) addStatic(n *discover.Node) { - // This overwites the task instead of updating an existing + // This overwrites the task instead of updating an existing // entry, giving users the opportunity to force a resolve operation. s.static[n.ID] = &dialTask{flags: staticDialedConn, dest: n} } From 92381ee00999398d0d0928d2c4be82b3334b912b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 23 Aug 2018 13:02:36 +0300 Subject: [PATCH 49/96] cmd, eth: clean up miner startup API, drop noop config field --- cmd/geth/main.go | 20 ++++-------- cmd/utils/flags.go | 6 ---- eth/api.go | 47 +++++++--------------------- eth/backend.go | 76 +++++++++++++++++++++++++++++++++++----------- eth/config.go | 1 - eth/gen_config.go | 5 --- 6 files changed, 75 insertions(+), 80 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 4b86382bdb..7b8a8244ee 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -336,26 +336,18 @@ func startNode(ctx *cli.Context, stack *node.Node) { if err := stack.Service(ðereum); err != nil { utils.Fatalf("Ethereum service not running: %v", err) } - // Use a reduced number of threads if requested - threads := ctx.GlobalInt(utils.MinerLegacyThreadsFlag.Name) - if ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) { - threads = ctx.GlobalInt(utils.MinerThreadsFlag.Name) - } - if threads > 0 { - type threaded interface { - SetThreads(threads int) - } - if th, ok := ethereum.Engine().(threaded); ok { - th.SetThreads(threads) - } - } // Set the gas price to the limits from the CLI and start mining gasprice := utils.GlobalBig(ctx, utils.MinerLegacyGasPriceFlag.Name) if ctx.IsSet(utils.MinerGasPriceFlag.Name) { gasprice = utils.GlobalBig(ctx, utils.MinerGasPriceFlag.Name) } ethereum.TxPool().SetGasPrice(gasprice) - if err := ethereum.StartMining(true); err != nil { + + threads := ctx.GlobalInt(utils.MinerLegacyThreadsFlag.Name) + if ctx.GlobalIsSet(utils.MinerThreadsFlag.Name) { + threads = ctx.GlobalInt(utils.MinerThreadsFlag.Name) + } + if err := ethereum.StartMining(threads); err != nil { utils.Fatalf("Failed to start mining: %v", err) } } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c9a936dd5f..b9a33ffe70 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1130,12 +1130,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheGCFlag.Name) { cfg.TrieCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheGCFlag.Name) / 100 } - if ctx.GlobalIsSet(MinerLegacyThreadsFlag.Name) { - cfg.MinerThreads = ctx.GlobalInt(MinerLegacyThreadsFlag.Name) - } - if ctx.GlobalIsSet(MinerThreadsFlag.Name) { - cfg.MinerThreads = ctx.GlobalInt(MinerThreadsFlag.Name) - } if ctx.GlobalIsSet(MinerNotifyFlag.Name) { cfg.MinerNotify = strings.Split(ctx.GlobalString(MinerNotifyFlag.Name), ",") } diff --git a/eth/api.go b/eth/api.go index 4b0ba8edbd..708f75a784 100644 --- a/eth/api.go +++ b/eth/api.go @@ -24,6 +24,7 @@ import ( "io" "math/big" "os" + "runtime" "strings" "time" @@ -34,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/internal/ethapi" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" @@ -94,47 +94,22 @@ func NewPrivateMinerAPI(e *Ethereum) *PrivateMinerAPI { return &PrivateMinerAPI{e: e} } -// Start the miner with the given number of threads. If threads is nil the number -// of workers started is equal to the number of logical CPUs that are usable by -// this process. If mining is already running, this method adjust the number of -// threads allowed to use and updates the minimum price required by the transaction -// pool. +// Start starts the miner with the given number of threads. If threads is nil, +// the number of workers started is equal to the number of logical CPUs that are +// usable by this process. If mining is already running, this method adjust the +// number of threads allowed to use and updates the minimum price required by the +// transaction pool. func (api *PrivateMinerAPI) Start(threads *int) error { - // Set the number of threads if the seal engine supports it if threads == nil { - threads = new(int) - } else if *threads == 0 { - *threads = -1 // Disable the miner from within + return api.e.StartMining(runtime.NumCPU()) } - type threaded interface { - SetThreads(threads int) - } - if th, ok := api.e.engine.(threaded); ok { - log.Info("Updated mining threads", "threads", *threads) - th.SetThreads(*threads) - } - // Start the miner and return - if !api.e.IsMining() { - // Propagate the initial price point to the transaction pool - api.e.lock.RLock() - price := api.e.gasPrice - api.e.lock.RUnlock() - api.e.txPool.SetGasPrice(price) - return api.e.StartMining(true) - } - return nil + return api.e.StartMining(*threads) } -// Stop the miner -func (api *PrivateMinerAPI) Stop() bool { - type threaded interface { - SetThreads(threads int) - } - if th, ok := api.e.engine.(threaded); ok { - th.SetThreads(-1) - } +// Stop terminates the miner, both at the consensus engine level as well as at +// the block creation level. +func (api *PrivateMinerAPI) Stop() { api.e.StopMining() - return true } // SetExtra sets the extra data string that is included when this miner mines a block. diff --git a/eth/backend.go b/eth/backend.go index 648175acfd..c530d3c7c3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -102,12 +102,18 @@ func (s *Ethereum) AddLesServer(ls LesServer) { // New creates a new Ethereum object (including the // initialisation of the common Ethereum object) func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { + // Ensure configuration values are compatible and sane if config.SyncMode == downloader.LightSync { return nil, errors.New("can't run eth.Ethereum in light sync mode, use les.LightEthereum") } if !config.SyncMode.IsValid() { return nil, fmt.Errorf("invalid sync mode %d", config.SyncMode) } + if config.MinerGasPrice == nil || config.MinerGasPrice.Cmp(common.Big0) <= 0 { + log.Warn("Sanitizing invalid miner gas price", "provided", config.MinerGasPrice, "updated", DefaultConfig.MinerGasPrice) + config.MinerGasPrice = new(big.Int).Set(DefaultConfig.MinerGasPrice) + } + // Assemble the Ethereum object chainDb, err := CreateDB(ctx, config, "chaindata") if err != nil { return nil, err @@ -333,32 +339,66 @@ func (s *Ethereum) SetEtherbase(etherbase common.Address) { s.miner.SetEtherbase(etherbase) } -func (s *Ethereum) StartMining(local bool) error { - eb, err := s.Etherbase() - if err != nil { - log.Error("Cannot start mining without etherbase", "err", err) - return fmt.Errorf("etherbase missing: %v", err) +// StartMining starts the miner with the given number of CPU threads. If mining +// is already running, this method adjust the number of threads allowed to use +// and updates the minimum price required by the transaction pool. +func (s *Ethereum) StartMining(threads int) error { + // Update the thread count within the consensus engine + type threaded interface { + SetThreads(threads int) } - if clique, ok := s.engine.(*clique.Clique); ok { - wallet, err := s.accountManager.Find(accounts.Account{Address: eb}) - if wallet == nil || err != nil { - log.Error("Etherbase account unavailable locally", "err", err) - return fmt.Errorf("signer missing: %v", err) + if th, ok := s.engine.(threaded); ok { + log.Info("Updated mining threads", "threads", threads) + if threads == 0 { + threads = -1 // Disable the miner from within } - clique.Authorize(eb, wallet.SignHash) + th.SetThreads(threads) } - if local { - // If local (CPU) mining is started, we can disable the transaction rejection - // mechanism introduced to speed sync times. CPU mining on mainnet is ludicrous - // so none will ever hit this path, whereas marking sync done on CPU mining - // will ensure that private networks work in single miner mode too. + // If the miner was not running, initialize it + if !s.IsMining() { + // Propagate the initial price point to the transaction pool + s.lock.RLock() + price := s.gasPrice + s.lock.RUnlock() + s.txPool.SetGasPrice(price) + + // Configure the local mining addess + eb, err := s.Etherbase() + if err != nil { + log.Error("Cannot start mining without etherbase", "err", err) + return fmt.Errorf("etherbase missing: %v", err) + } + if clique, ok := s.engine.(*clique.Clique); ok { + wallet, err := s.accountManager.Find(accounts.Account{Address: eb}) + if wallet == nil || err != nil { + log.Error("Etherbase account unavailable locally", "err", err) + return fmt.Errorf("signer missing: %v", err) + } + clique.Authorize(eb, wallet.SignHash) + } + // If mining is started, we can disable the transaction rejection mechanism + // introduced to speed sync times. atomic.StoreUint32(&s.protocolManager.acceptTxs, 1) + + go s.miner.Start(eb) } - go s.miner.Start(eb) return nil } -func (s *Ethereum) StopMining() { s.miner.Stop() } +// StopMining terminates the miner, both at the consensus engine level as well as +// at the block creation level. +func (s *Ethereum) StopMining() { + // Update the thread count within the consensus engine + type threaded interface { + SetThreads(threads int) + } + if th, ok := s.engine.(threaded); ok { + th.SetThreads(-1) + } + // Stop the block creating itself + s.miner.Stop() +} + func (s *Ethereum) IsMining() bool { return s.miner.Mining() } func (s *Ethereum) Miner() *miner.Miner { return s.miner } diff --git a/eth/config.go b/eth/config.go index 1398a55145..5b2eca585c 100644 --- a/eth/config.go +++ b/eth/config.go @@ -97,7 +97,6 @@ type Config struct { // Mining-related options Etherbase common.Address `toml:",omitempty"` - MinerThreads int `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData []byte `toml:",omitempty"` MinerGasPrice *big.Int diff --git a/eth/gen_config.go b/eth/gen_config.go index 62556be7e1..a72e35bcd1 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -31,7 +31,6 @@ func (c Config) MarshalTOML() (interface{}, error) { TrieCache int TrieTimeout time.Duration Etherbase common.Address `toml:",omitempty"` - MinerThreads int `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData hexutil.Bytes `toml:",omitempty"` MinerGasPrice *big.Int @@ -55,7 +54,6 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieCache = c.TrieCache enc.TrieTimeout = c.TrieTimeout enc.Etherbase = c.Etherbase - enc.MinerThreads = c.MinerThreads enc.MinerNotify = c.MinerNotify enc.MinerExtraData = c.MinerExtraData enc.MinerGasPrice = c.MinerGasPrice @@ -134,9 +132,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.Etherbase != nil { c.Etherbase = *dec.Etherbase } - if dec.MinerThreads != nil { - c.MinerThreads = *dec.MinerThreads - } if dec.MinerNotify != nil { c.MinerNotify = dec.MinerNotify } From 1e63a015a56ae45825c842b5a2b17d2cf2ac1ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 23 Aug 2018 14:03:13 +0300 Subject: [PATCH 50/96] miner: add two stress tests based on clique and ethash --- miner/stress_clique.go | 217 +++++++++++++++++++++++++++++++++++++++++ miner/stress_ethash.go | 197 +++++++++++++++++++++++++++++++++++++ 2 files changed, 414 insertions(+) create mode 100644 miner/stress_clique.go create mode 100644 miner/stress_ethash.go diff --git a/miner/stress_clique.go b/miner/stress_clique.go new file mode 100644 index 0000000000..8545b9413a --- /dev/null +++ b/miner/stress_clique.go @@ -0,0 +1,217 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build none + +// This file contains a miner stress test based on the Clique consensus engine. +package main + +import ( + "bytes" + "crypto/ecdsa" + "fmt" + "io/ioutil" + "math/big" + "math/rand" + "os" + "time" + + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/params" +) + +func main() { + log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + fdlimit.Raise(2048) + + // Generate a batch of accounts to seal and fund with + faucets := make([]*ecdsa.PrivateKey, 128) + for i := 0; i < len(faucets); i++ { + faucets[i], _ = crypto.GenerateKey() + } + sealers := make([]*ecdsa.PrivateKey, 4) + for i := 0; i < len(sealers); i++ { + sealers[i], _ = crypto.GenerateKey() + } + // Create a Clique network based off of the Rinkeby config + genesis := makeGenesis(faucets, sealers) + + var ( + nodes []*node.Node + enodes []string + ) + for _, sealer := range sealers { + // Start the node and wait until it's up + node, err := makeSealer(genesis, enodes) + if err != nil { + panic(err) + } + defer node.Stop() + + for node.Server().NodeInfo().Ports.Listener == 0 { + time.Sleep(250 * time.Millisecond) + } + // Connect the node to al the previous ones + for _, enode := range enodes { + enode, err := discover.ParseNode(enode) + if err != nil { + panic(err) + } + node.Server().AddPeer(enode) + } + // Start tracking the node and it's enode url + nodes = append(nodes, node) + + enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener) + enodes = append(enodes, enode) + + // Inject the signer key and start sealing with it + store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) + signer, err := store.ImportECDSA(sealer, "") + if err != nil { + panic(err) + } + if err := store.Unlock(signer, ""); err != nil { + panic(err) + } + } + // Iterate over all the nodes and start signing with them + time.Sleep(3 * time.Second) + + for _, node := range nodes { + var ethereum *eth.Ethereum + if err := node.Service(ðereum); err != nil { + panic(err) + } + if err := ethereum.StartMining(1); err != nil { + panic(err) + } + } + time.Sleep(3 * time.Second) + + // Start injecting transactions from the faucet like crazy + nonces := make([]uint64, len(faucets)) + for { + index := rand.Intn(len(faucets)) + + // Fetch the accessor for the relevant signer + var ethereum *eth.Ethereum + if err := nodes[index%len(nodes)].Service(ðereum); err != nil { + panic(err) + } + // Create a self transaction and inject into the pool + tx, err := types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000), nil), types.HomesteadSigner{}, faucets[index]) + if err != nil { + panic(err) + } + if err := ethereum.TxPool().AddLocal(tx); err != nil { + panic(err) + } + nonces[index]++ + + // Wait if we're too saturated + if pend, _ := ethereum.TxPool().Stats(); pend > 2048 { + time.Sleep(100 * time.Millisecond) + } + } +} + +// makeGenesis creates a custom Clique genesis block based on some pre-defined +// signer and faucet accounts. +func makeGenesis(faucets []*ecdsa.PrivateKey, sealers []*ecdsa.PrivateKey) *core.Genesis { + // Create a Clique network based off of the Rinkeby config + genesis := core.DefaultRinkebyGenesisBlock() + genesis.GasLimit = 25000000 + + genesis.Config.ChainID = big.NewInt(18) + genesis.Config.Clique.Period = 1 + genesis.Config.EIP150Hash = common.Hash{} + + genesis.Alloc = core.GenesisAlloc{} + for _, faucet := range faucets { + genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = core.GenesisAccount{ + Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), + } + } + // Sort the signers and embed into the extra-data section + signers := make([]common.Address, len(sealers)) + for i, sealer := range sealers { + signers[i] = crypto.PubkeyToAddress(sealer.PublicKey) + } + for i := 0; i < len(signers); i++ { + for j := i + 1; j < len(signers); j++ { + if bytes.Compare(signers[i][:], signers[j][:]) > 0 { + signers[i], signers[j] = signers[j], signers[i] + } + } + } + genesis.ExtraData = make([]byte, 32+len(signers)*common.AddressLength+65) + for i, signer := range signers { + copy(genesis.ExtraData[32+i*common.AddressLength:], signer[:]) + } + // Return the genesis block for initialization + return genesis +} + +func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) { + // Define the basic configurations for the Ethereum node + datadir, _ := ioutil.TempDir("", "") + + config := &node.Config{ + Name: "geth", + Version: params.Version, + DataDir: datadir, + P2P: p2p.Config{ + ListenAddr: "0.0.0.0:0", + NoDiscovery: true, + MaxPeers: 25, + }, + NoUSB: true, + } + // Start the node and configure a full Ethereum node on it + stack, err := node.New(config) + if err != nil { + return nil, err + } + if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + return eth.New(ctx, ð.Config{ + Genesis: genesis, + NetworkId: genesis.Config.ChainID.Uint64(), + SyncMode: downloader.FullSync, + DatabaseCache: 256, + DatabaseHandles: 256, + TxPool: core.DefaultTxPoolConfig, + GPO: eth.DefaultConfig.GPO, + MinerGasPrice: big.NewInt(1), + MinerRecommit: time.Second, + }) + }); err != nil { + return nil, err + } + // Start the node and return if successful + return stack, stack.Start() +} diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go new file mode 100644 index 0000000000..4ed8aeee4f --- /dev/null +++ b/miner/stress_ethash.go @@ -0,0 +1,197 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build none + +// This file contains a miner stress test based on the Ethash consensus engine. +package main + +import ( + "crypto/ecdsa" + "fmt" + "io/ioutil" + "math/big" + "math/rand" + "os" + "path/filepath" + "time" + + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/params" +) + +func main() { + log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + fdlimit.Raise(2048) + + // Generate a batch of accounts to seal and fund with + faucets := make([]*ecdsa.PrivateKey, 128) + for i := 0; i < len(faucets); i++ { + faucets[i], _ = crypto.GenerateKey() + } + // Pre-generate the ethash mining DAG so we don't race + ethash.MakeDataset(1, filepath.Join(os.Getenv("HOME"), ".ethash")) + + // Create an Ethash network based off of the Ropsten config + genesis := makeGenesis(faucets) + + var ( + nodes []*node.Node + enodes []string + ) + for i := 0; i < 4; i++ { + // Start the node and wait until it's up + node, err := makeMiner(genesis, enodes) + if err != nil { + panic(err) + } + defer node.Stop() + + for node.Server().NodeInfo().Ports.Listener == 0 { + time.Sleep(250 * time.Millisecond) + } + // Connect the node to al the previous ones + for _, enode := range enodes { + enode, err := discover.ParseNode(enode) + if err != nil { + panic(err) + } + node.Server().AddPeer(enode) + } + // Start tracking the node and it's enode url + nodes = append(nodes, node) + + enode := fmt.Sprintf("enode://%s@127.0.0.1:%d", node.Server().NodeInfo().ID, node.Server().NodeInfo().Ports.Listener) + enodes = append(enodes, enode) + + // Inject the signer key and start sealing with it + store := node.AccountManager().Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) + if _, err := store.NewAccount(""); err != nil { + panic(err) + } + } + // Iterate over all the nodes and start signing with them + time.Sleep(3 * time.Second) + + for _, node := range nodes { + var ethereum *eth.Ethereum + if err := node.Service(ðereum); err != nil { + panic(err) + } + if err := ethereum.StartMining(1); err != nil { + panic(err) + } + } + time.Sleep(3 * time.Second) + + // Start injecting transactions from the faucets like crazy + nonces := make([]uint64, len(faucets)) + for { + index := rand.Intn(len(faucets)) + + // Fetch the accessor for the relevant signer + var ethereum *eth.Ethereum + if err := nodes[index%len(nodes)].Service(ðereum); err != nil { + panic(err) + } + // Create a self transaction and inject into the pool + tx, err := types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000+rand.Int63n(65536)), nil), types.HomesteadSigner{}, faucets[index]) + if err != nil { + panic(err) + } + if err := ethereum.TxPool().AddLocal(tx); err != nil { + panic(err) + } + nonces[index]++ + + // Wait if we're too saturated + if pend, _ := ethereum.TxPool().Stats(); pend > 2048 { + time.Sleep(100 * time.Millisecond) + } + } +} + +// makeGenesis creates a custom Ethash genesis block based on some pre-defined +// faucet accounts. +func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis { + genesis := core.DefaultTestnetGenesisBlock() + genesis.Difficulty = params.MinimumDifficulty + genesis.GasLimit = 25000000 + + genesis.Config.ChainID = big.NewInt(18) + genesis.Config.EIP150Hash = common.Hash{} + + genesis.Alloc = core.GenesisAlloc{} + for _, faucet := range faucets { + genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = core.GenesisAccount{ + Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), + } + } + return genesis +} + +func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) { + // Define the basic configurations for the Ethereum node + datadir, _ := ioutil.TempDir("", "") + + config := &node.Config{ + Name: "geth", + Version: params.Version, + DataDir: datadir, + P2P: p2p.Config{ + ListenAddr: "0.0.0.0:0", + NoDiscovery: true, + MaxPeers: 25, + }, + NoUSB: true, + UseLightweightKDF: true, + } + // Start the node and configure a full Ethereum node on it + stack, err := node.New(config) + if err != nil { + return nil, err + } + if err := stack.Register(func(ctx *node.ServiceContext) (node.Service, error) { + return eth.New(ctx, ð.Config{ + Genesis: genesis, + NetworkId: genesis.Config.ChainID.Uint64(), + SyncMode: downloader.FullSync, + DatabaseCache: 256, + DatabaseHandles: 256, + TxPool: core.DefaultTxPoolConfig, + GPO: eth.DefaultConfig.GPO, + Ethash: eth.DefaultConfig.Ethash, + MinerGasPrice: big.NewInt(1), + MinerRecommit: time.Second, + }) + }); err != nil { + return nil, err + } + // Start the node and return if successful + return stack, stack.Start() +} From 1136269a79e6ee8bc97f5bf277bf8ec12286b79b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 23 Aug 2018 15:44:27 +0300 Subject: [PATCH 51/96] miner: differentiate between uncle and lost block --- miner/unconfirmed.go | 35 +++++++++++++++++++++++++++-------- miner/unconfirmed_test.go | 13 ++++++++----- miner/worker.go | 2 +- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/miner/unconfirmed.go b/miner/unconfirmed.go index 7a4fc7cc5a..3eb29bc705 100644 --- a/miner/unconfirmed.go +++ b/miner/unconfirmed.go @@ -25,11 +25,14 @@ import ( "github.com/ethereum/go-ethereum/log" ) -// headerRetriever is used by the unconfirmed block set to verify whether a previously +// chainRetriever is used by the unconfirmed block set to verify whether a previously // mined block is part of the canonical chain or not. -type headerRetriever interface { +type chainRetriever interface { // GetHeaderByNumber retrieves the canonical header associated with a block number. GetHeaderByNumber(number uint64) *types.Header + + // GetBlockByNumber retrieves the canonical block associated with a block number. + GetBlockByNumber(number uint64) *types.Block } // unconfirmedBlock is a small collection of metadata about a locally mined block @@ -44,14 +47,14 @@ type unconfirmedBlock struct { // used by the miner to provide logs to the user when a previously mined block // has a high enough guarantee to not be reorged out of the canonical chain. type unconfirmedBlocks struct { - chain headerRetriever // Blockchain to verify canonical status through - depth uint // Depth after which to discard previous blocks - blocks *ring.Ring // Block infos to allow canonical chain cross checks - lock sync.RWMutex // Protects the fields from concurrent access + chain chainRetriever // Blockchain to verify canonical status through + depth uint // Depth after which to discard previous blocks + blocks *ring.Ring // Block infos to allow canonical chain cross checks + lock sync.RWMutex // Protects the fields from concurrent access } // newUnconfirmedBlocks returns new data structure to track currently unconfirmed blocks. -func newUnconfirmedBlocks(chain headerRetriever, depth uint) *unconfirmedBlocks { +func newUnconfirmedBlocks(chain chainRetriever, depth uint) *unconfirmedBlocks { return &unconfirmedBlocks{ chain: chain, depth: depth, @@ -103,7 +106,23 @@ func (set *unconfirmedBlocks) Shift(height uint64) { case header.Hash() == next.hash: log.Info("🔗 block reached canonical chain", "number", next.index, "hash", next.hash) default: - log.Info("â‘‚ block became a side fork", "number", next.index, "hash", next.hash) + // Block is not canonical, check whether we have an uncle or a lost block + included := false + for number := next.index; !included && number < next.index+uint64(set.depth) && number <= height; number++ { + if block := set.chain.GetBlockByNumber(number); block != nil { + for _, uncle := range block.Uncles() { + if uncle.Hash() == next.hash { + included = true + break + } + } + } + } + if included { + log.Info("â‘‚ block became an uncle", "number", next.index, "hash", next.hash) + } else { + log.Info("😱 block lost", "number", next.index, "hash", next.hash) + } } // Drop the block out of the ring if set.blocks.Value == set.blocks.Next().Value { diff --git a/miner/unconfirmed_test.go b/miner/unconfirmed_test.go index 456af1764a..42e77f3e64 100644 --- a/miner/unconfirmed_test.go +++ b/miner/unconfirmed_test.go @@ -23,11 +23,14 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// noopHeaderRetriever is an implementation of headerRetriever that always +// noopChainRetriever is an implementation of headerRetriever that always // returns nil for any requested headers. -type noopHeaderRetriever struct{} +type noopChainRetriever struct{} -func (r *noopHeaderRetriever) GetHeaderByNumber(number uint64) *types.Header { +func (r *noopChainRetriever) GetHeaderByNumber(number uint64) *types.Header { + return nil +} +func (r *noopChainRetriever) GetBlockByNumber(number uint64) *types.Block { return nil } @@ -36,7 +39,7 @@ func (r *noopHeaderRetriever) GetHeaderByNumber(number uint64) *types.Header { func TestUnconfirmedInsertBounds(t *testing.T) { limit := uint(10) - pool := newUnconfirmedBlocks(new(noopHeaderRetriever), limit) + pool := newUnconfirmedBlocks(new(noopChainRetriever), limit) for depth := uint64(0); depth < 2*uint64(limit); depth++ { // Insert multiple blocks for the same level just to stress it for i := 0; i < int(depth); i++ { @@ -58,7 +61,7 @@ func TestUnconfirmedShifts(t *testing.T) { // Create a pool with a few blocks on various depths limit, start := uint(10), uint64(25) - pool := newUnconfirmedBlocks(new(noopHeaderRetriever), limit) + pool := newUnconfirmedBlocks(new(noopChainRetriever), limit) for depth := start; depth < start+uint64(limit); depth++ { pool.Insert(depth, common.Hash([32]byte{byte(depth)})) } diff --git a/miner/worker.go b/miner/worker.go index 8c3337ba45..cedc158b51 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -55,7 +55,7 @@ const ( resubmitAdjustChanSize = 10 // miningLogAtDepth is the number of confirmations before logging successful mining. - miningLogAtDepth = 5 + miningLogAtDepth = 7 // minRecommitInterval is the minimal time interval to recreate the mining block with // any newly arrived transactions. From c3f7e3be3b60df3edd168e80aa89ee2992932b0d Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 23 Aug 2018 20:59:58 +0800 Subject: [PATCH 52/96] core/statedb: deep copy logs (#17489) --- core/state/statedb.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index 92d394ae32..101b03a127 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -489,10 +489,13 @@ func (self *StateDB) Copy() *StateDB { state.stateObjectsDirty[addr] = struct{}{} } } - for hash, logs := range self.logs { - state.logs[hash] = make([]*types.Log, len(logs)) - copy(state.logs[hash], logs) + cpy := make([]*types.Log, len(logs)) + for i, l := range logs { + cpy[i] = new(types.Log) + *cpy[i] = *l + } + state.logs[hash] = cpy } for hash, preimage := range self.preimages { state.preimages[hash] = preimage From 40a71f28cf1ada0bf6bdcdc2f3c6f31a8da134a2 Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 23 Aug 2018 21:02:57 +0800 Subject: [PATCH 53/96] miner: fix state commit, track old work packages too (#17490) * miner: commit state which is relative with sealing result * consensus, core, miner, mobile: introduce sealHash interface * miner: evict pending task with threshold * miner: go fmt --- consensus/clique/clique.go | 5 +++ consensus/consensus.go | 3 ++ consensus/ethash/consensus.go | 29 ++++++++++++- consensus/ethash/ethash_test.go | 11 +++-- consensus/ethash/sealer.go | 4 +- consensus/ethash/sealer_test.go | 2 +- core/types/block.go | 23 ----------- miner/worker.go | 73 +++++++++++++++++++++++---------- mobile/types.go | 35 ++++++++-------- 9 files changed, 112 insertions(+), 73 deletions(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 0859447014..3730c91f62 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -673,6 +673,11 @@ func CalcDifficulty(snap *Snapshot, signer common.Address) *big.Int { return new(big.Int).Set(diffNoTurn) } +// SealHash returns the hash of a block prior to it being sealed. +func (c *Clique) SealHash(header *types.Header) common.Hash { + return sigHash(header) +} + // Close implements consensus.Engine. It's a noop for clique as there is are no background threads. func (c *Clique) Close() error { return nil diff --git a/consensus/consensus.go b/consensus/consensus.go index 8271754445..27799f13c0 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -90,6 +90,9 @@ type Engine interface { // seal place on top. Seal(chain ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) + // SealHash returns the hash of a block prior to it being sealed. + SealHash(header *types.Header) common.Hash + // CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty // that a new block should have. CalcDifficulty(chain ChainReader, time uint64, parent *types.Header) *big.Int diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 86fd997ae4..259cc5056b 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -31,7 +31,9 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" ) // Ethash proof-of-work protocol constants. @@ -495,7 +497,7 @@ func (ethash *Ethash) verifySeal(chain consensus.ChainReader, header *types.Head if fulldag { dataset := ethash.dataset(number, true) if dataset.generated() { - digest, result = hashimotoFull(dataset.dataset, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + digest, result = hashimotoFull(dataset.dataset, ethash.SealHash(header).Bytes(), header.Nonce.Uint64()) // Datasets are unmapped in a finalizer. Ensure that the dataset stays alive // until after the call to hashimotoFull so it's not unmapped while being used. @@ -513,7 +515,7 @@ func (ethash *Ethash) verifySeal(chain consensus.ChainReader, header *types.Head if ethash.config.PowMode == ModeTest { size = 32 * 1024 } - digest, result = hashimotoLight(size, cache.cache, header.HashNoNonce().Bytes(), header.Nonce.Uint64()) + digest, result = hashimotoLight(size, cache.cache, ethash.SealHash(header).Bytes(), header.Nonce.Uint64()) // Caches are unmapped in a finalizer. Ensure that the cache stays alive // until after the call to hashimotoLight so it's not unmapped while being used. @@ -552,6 +554,29 @@ func (ethash *Ethash) Finalize(chain consensus.ChainReader, header *types.Header return types.NewBlock(header, txs, uncles, receipts), nil } +// SealHash returns the hash of a block prior to it being sealed. +func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { + hasher := sha3.NewKeccak256() + + rlp.Encode(hasher, []interface{}{ + header.ParentHash, + header.UncleHash, + header.Coinbase, + header.Root, + header.TxHash, + header.ReceiptHash, + header.Bloom, + header.Difficulty, + header.Number, + header.GasLimit, + header.GasUsed, + header.Time, + header.Extra, + }) + hasher.Sum(hash[:0]) + return hash +} + // Some weird constants to avoid constant memory allocs for them. var ( big8 = big.NewInt(8) diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index 87ac17c2b2..b190d63d6b 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -94,6 +94,7 @@ func TestRemoteSealer(t *testing.T) { } header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} block := types.NewBlockWithHeader(header) + sealhash := ethash.SealHash(header) // Push new work. ethash.Seal(nil, block, nil) @@ -102,27 +103,29 @@ func TestRemoteSealer(t *testing.T) { work [3]string err error ) - if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() { + if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() { t.Error("expect to return a mining work has same hash") } - if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res { + if res := api.SubmitWork(types.BlockNonce{}, sealhash, common.Hash{}); res { t.Error("expect to return false when submit a fake solution") } // Push new block with same block number to replace the original one. header = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)} block = types.NewBlockWithHeader(header) + sealhash = ethash.SealHash(header) ethash.Seal(nil, block, nil) - if work, err = api.GetWork(); err != nil || work[0] != block.HashNoNonce().Hex() { + if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() { t.Error("expect to return the latest pushed work") } // Push block with higher block number. newHead := &types.Header{Number: big.NewInt(2), Difficulty: big.NewInt(100)} newBlock := types.NewBlockWithHeader(newHead) + newSealhash := ethash.SealHash(newHead) ethash.Seal(nil, newBlock, nil) - if res := api.SubmitWork(types.BlockNonce{}, block.HashNoNonce(), common.Hash{}); res { + if res := api.SubmitWork(types.BlockNonce{}, newSealhash, common.Hash{}); res { t.Error("expect to return false when submit a stale solution") } } diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index c3b2c86d12..a458c60f66 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -111,7 +111,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s // Extract some data from the header var ( header = block.Header() - hash = header.HashNoNonce().Bytes() + hash = ethash.SealHash(header).Bytes() target = new(big.Int).Div(two256, header.Difficulty) number = header.Number.Uint64() dataset = ethash.dataset(number, false) @@ -213,7 +213,7 @@ func (ethash *Ethash) remote(notify []string) { // result[1], 32 bytes hex encoded seed hash used for DAG // result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty makeWork := func(block *types.Block) { - hash := block.HashNoNonce() + hash := ethash.SealHash(block.Header()) currentWork[0] = hash.Hex() currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex() diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go index 6c7157a5ae..d1b66f9cf3 100644 --- a/consensus/ethash/sealer_test.go +++ b/consensus/ethash/sealer_test.go @@ -51,7 +51,7 @@ func TestRemoteNotify(t *testing.T) { ethash.Seal(nil, block, nil) select { case work := <-sink: - if want := header.HashNoNonce().Hex(); work[0] != want { + if want := ethash.SealHash(header).Hex(); work[0] != want { t.Errorf("work packet hash mismatch: have %s, want %s", work[0], want) } if want := common.BytesToHash(SeedHash(header.Number.Uint64())).Hex(); work[1] != want { diff --git a/core/types/block.go b/core/types/block.go index ae1b4299d3..8a21bba1e7 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -102,25 +102,6 @@ func (h *Header) Hash() common.Hash { return rlpHash(h) } -// HashNoNonce returns the hash which is used as input for the proof-of-work search. -func (h *Header) HashNoNonce() common.Hash { - return rlpHash([]interface{}{ - h.ParentHash, - h.UncleHash, - h.Coinbase, - h.Root, - h.TxHash, - h.ReceiptHash, - h.Bloom, - h.Difficulty, - h.Number, - h.GasLimit, - h.GasUsed, - h.Time, - h.Extra, - }) -} - // Size returns the approximate memory used by all internal contents. It is used // to approximate and limit the memory consumption of various caches. func (h *Header) Size() common.StorageSize { @@ -324,10 +305,6 @@ func (b *Block) Header() *Header { return CopyHeader(b.header) } // Body returns the non-header content of the block. func (b *Block) Body() *Body { return &Body{b.transactions, b.uncles} } -func (b *Block) HashNoNonce() common.Hash { - return b.header.HashNoNonce() -} - // Size returns the true RLP encoded storage size of the block, either by encoding // and returning it, or returning a previsouly cached value. func (b *Block) Size() common.StorageSize { diff --git a/miner/worker.go b/miner/worker.go index 8c3337ba45..18fb12e451 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -72,6 +72,9 @@ const ( // intervalAdjustBias is applied during the new resubmit interval calculation in favor of // increasing upper limit or decreasing lower limit so that the limit can be reachable. intervalAdjustBias = 200 * 1000.0 * 1000.0 + + // staleThreshold is the maximum distance of the acceptable stale block. + staleThreshold = 7 ) // environment is the worker's current environment and holds all of the current state information. @@ -150,6 +153,9 @@ type worker struct { coinbase common.Address extra []byte + pendingMu sync.RWMutex + pendingTasks map[common.Hash]*task + snapshotMu sync.RWMutex // The lock used to protect the block snapshot and state snapshot snapshotBlock *types.Block snapshotState *state.StateDB @@ -174,6 +180,7 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, chain: eth.BlockChain(), possibleUncles: make(map[common.Hash]*types.Block), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), + pendingTasks: make(map[common.Hash]*task), txsCh: make(chan core.NewTxsEvent, txChanSize), chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), @@ -317,13 +324,25 @@ func (w *worker) newWorkLoop(recommit time.Duration) { } recommit = time.Duration(int64(next)) } + // clearPending cleans the stale pending tasks. + clearPending := func(number uint64) { + w.pendingMu.Lock() + for h, t := range w.pendingTasks { + if t.block.NumberU64()+staleThreshold <= number { + delete(w.pendingTasks, h) + } + } + w.pendingMu.Unlock() + } for { select { case <-w.startCh: + clearPending(w.chain.CurrentBlock().NumberU64()) commit(false, commitInterruptNewHead) - case <-w.chainHeadCh: + case head := <-w.chainHeadCh: + clearPending(head.Block.NumberU64()) commit(false, commitInterruptNewHead) case <-timer.C: @@ -454,28 +473,37 @@ func (w *worker) mainLoop() { // seal pushes a sealing task to consensus engine and submits the result. func (w *worker) seal(t *task, stop <-chan struct{}) { - var ( - err error - res *task - ) - if w.skipSealHook != nil && w.skipSealHook(t) { return } + // The reason for caching task first is: + // A previous sealing action will be canceled by subsequent actions, + // however, remote miner may submit a result based on the cancelled task. + // So we should only submit the pending state corresponding to the seal result. + // TODO(rjl493456442) Replace the seal-wait logic structure + w.pendingMu.Lock() + w.pendingTasks[w.engine.SealHash(t.block.Header())] = t + w.pendingMu.Unlock() - if t.block, err = w.engine.Seal(w.chain, t.block, stop); t.block != nil { - log.Info("Successfully sealed new block", "number", t.block.Number(), "hash", t.block.Hash(), - "elapsed", common.PrettyDuration(time.Since(t.createdAt))) - res = t - } else { - if err != nil { - log.Warn("Block sealing failed", "err", err) + if block, err := w.engine.Seal(w.chain, t.block, stop); block != nil { + sealhash := w.engine.SealHash(block.Header()) + w.pendingMu.RLock() + task, exist := w.pendingTasks[sealhash] + w.pendingMu.RUnlock() + if !exist { + log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", block.Hash()) + return } - res = nil - } - select { - case w.resultCh <- res: - case <-w.exitCh: + // Assemble sealing result + task.block = block + log.Info("Successfully sealed new block", "number", block.Number(), "sealhash", sealhash, "hash", block.Hash(), + "elapsed", common.PrettyDuration(time.Since(task.createdAt))) + select { + case w.resultCh <- task: + case <-w.exitCh: + } + } else if err != nil { + log.Warn("Block sealing failed", "err", err) } } @@ -501,12 +529,13 @@ func (w *worker) taskLoop() { w.newTaskHook(task) } // Reject duplicate sealing work due to resubmitting. - if task.block.HashNoNonce() == prev { + sealHash := w.engine.SealHash(task.block.Header()) + if sealHash == prev { continue } interrupt() stopCh = make(chan struct{}) - prev = task.block.HashNoNonce() + prev = sealHash go w.seal(task, stopCh) case <-w.exitCh: interrupt() @@ -928,8 +957,8 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st } feesEth := new(big.Float).Quo(new(big.Float).SetInt(feesWei), new(big.Float).SetInt(big.NewInt(params.Ether))) - log.Info("Commit new mining work", "number", block.Number(), "uncles", len(uncles), "txs", w.current.tcount, - "gas", block.GasUsed(), "fees", feesEth, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Commit new mining work", "number", block.Number(), "sealhash", w.engine.SealHash(block.Header()), + "uncles", len(uncles), "txs", w.current.tcount, "gas", block.GasUsed(), "fees", feesEth, "elapsed", common.PrettyDuration(time.Since(start))) case <-w.exitCh: log.Info("Worker has exited") diff --git a/mobile/types.go b/mobile/types.go index f32b4918f3..b2780f3076 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -168,25 +168,22 @@ func (b *Block) EncodeJSON() (string, error) { return string(data), err } -func (b *Block) GetParentHash() *Hash { return &Hash{b.block.ParentHash()} } -func (b *Block) GetUncleHash() *Hash { return &Hash{b.block.UncleHash()} } -func (b *Block) GetCoinbase() *Address { return &Address{b.block.Coinbase()} } -func (b *Block) GetRoot() *Hash { return &Hash{b.block.Root()} } -func (b *Block) GetTxHash() *Hash { return &Hash{b.block.TxHash()} } -func (b *Block) GetReceiptHash() *Hash { return &Hash{b.block.ReceiptHash()} } -func (b *Block) GetBloom() *Bloom { return &Bloom{b.block.Bloom()} } -func (b *Block) GetDifficulty() *BigInt { return &BigInt{b.block.Difficulty()} } -func (b *Block) GetNumber() int64 { return b.block.Number().Int64() } -func (b *Block) GetGasLimit() int64 { return int64(b.block.GasLimit()) } -func (b *Block) GetGasUsed() int64 { return int64(b.block.GasUsed()) } -func (b *Block) GetTime() int64 { return b.block.Time().Int64() } -func (b *Block) GetExtra() []byte { return b.block.Extra() } -func (b *Block) GetMixDigest() *Hash { return &Hash{b.block.MixDigest()} } -func (b *Block) GetNonce() int64 { return int64(b.block.Nonce()) } - -func (b *Block) GetHash() *Hash { return &Hash{b.block.Hash()} } -func (b *Block) GetHashNoNonce() *Hash { return &Hash{b.block.HashNoNonce()} } - +func (b *Block) GetParentHash() *Hash { return &Hash{b.block.ParentHash()} } +func (b *Block) GetUncleHash() *Hash { return &Hash{b.block.UncleHash()} } +func (b *Block) GetCoinbase() *Address { return &Address{b.block.Coinbase()} } +func (b *Block) GetRoot() *Hash { return &Hash{b.block.Root()} } +func (b *Block) GetTxHash() *Hash { return &Hash{b.block.TxHash()} } +func (b *Block) GetReceiptHash() *Hash { return &Hash{b.block.ReceiptHash()} } +func (b *Block) GetBloom() *Bloom { return &Bloom{b.block.Bloom()} } +func (b *Block) GetDifficulty() *BigInt { return &BigInt{b.block.Difficulty()} } +func (b *Block) GetNumber() int64 { return b.block.Number().Int64() } +func (b *Block) GetGasLimit() int64 { return int64(b.block.GasLimit()) } +func (b *Block) GetGasUsed() int64 { return int64(b.block.GasUsed()) } +func (b *Block) GetTime() int64 { return b.block.Time().Int64() } +func (b *Block) GetExtra() []byte { return b.block.Extra() } +func (b *Block) GetMixDigest() *Hash { return &Hash{b.block.MixDigest()} } +func (b *Block) GetNonce() int64 { return int64(b.block.Nonce()) } +func (b *Block) GetHash() *Hash { return &Hash{b.block.Hash()} } func (b *Block) GetHeader() *Header { return &Header{b.block.Header()} } func (b *Block) GetUncles() *Headers { return &Headers{b.block.Uncles()} } func (b *Block) GetTransactions() *Transactions { return &Transactions{b.block.Transactions()} } From 70398d300d4da97c89f96f5c9629caa327de5c39 Mon Sep 17 00:00:00 2001 From: Mymskmkt <1847234666@qq.com> Date: Sat, 25 Aug 2018 02:08:48 +0800 Subject: [PATCH 54/96] trie: fix typo (#17498) --- trie/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trie/sync.go b/trie/sync.go index ccec80c9e3..88d6eb7799 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -73,7 +73,7 @@ func newSyncMemBatch() *syncMemBatch { // and reconstructs the trie step by step until all is done. type Sync struct { database DatabaseReader // Persistent database to check for existing entries - membatch *syncMemBatch // Memory buffer to avoid frequest database writes + membatch *syncMemBatch // Memory buffer to avoid frequent database writes requests map[common.Hash]*request // Pending requests pertaining to a key hash queue *prque.Prque // Priority queue with the pending requests } From d1aa605f1e8639769cdd75bcec3064c29a62b34a Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Mon, 27 Aug 2018 16:49:29 +0800 Subject: [PATCH 55/96] all: remove the duplicate 'the' in annotations (#17509) --- accounts/abi/bind/bind.go | 4 ++-- cmd/wnode/main.go | 2 +- common/bitutil/compress_test.go | 4 ++-- console/prompter.go | 4 ++-- core/block_validator.go | 2 +- core/state_processor.go | 2 +- core/vm/contract.go | 2 +- crypto/secp256k1/secp256.go | 2 +- eth/downloader/queue.go | 2 +- eth/fetcher/fetcher.go | 2 +- p2p/server.go | 2 +- swarm/api/api.go | 2 +- swarm/network/README.md | 2 +- swarm/network/stream/intervals/intervals.go | 2 +- swarm/network/stream/snapshot_sync_test.go | 2 +- whisper/whisperv5/whisper.go | 2 +- whisper/whisperv6/whisper.go | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 411177057b..172bed8fa0 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -207,7 +207,7 @@ func bindTypeGo(kind abi.Type) string { // The inner function of bindTypeGo, this finds the inner type of stringKind. // (Or just the type itself if it is not an array or slice) -// The length of the matched part is returned, with the the translated type. +// The length of the matched part is returned, with the translated type. func bindUnnestedTypeGo(stringKind string) (int, string) { switch { @@ -255,7 +255,7 @@ func bindTypeJava(kind abi.Type) string { // The inner function of bindTypeJava, this finds the inner type of stringKind. // (Or just the type itself if it is not an array or slice) -// The length of the matched part is returned, with the the translated type. +// The length of the matched part is returned, with the translated type. func bindUnnestedTypeJava(stringKind string) (int, string) { switch { diff --git a/cmd/wnode/main.go b/cmd/wnode/main.go index 31b65c1da9..ae79841f6b 100644 --- a/cmd/wnode/main.go +++ b/cmd/wnode/main.go @@ -754,7 +754,7 @@ func extractIDFromEnode(s string) []byte { return n.ID[:] } -// obfuscateBloom adds 16 random bits to the the bloom +// obfuscateBloom adds 16 random bits to the bloom // filter, in order to obfuscate the containing topics. // it does so deterministically within every session. // despite additional bits, it will match on average diff --git a/common/bitutil/compress_test.go b/common/bitutil/compress_test.go index 9bd1de103a..13a13011dc 100644 --- a/common/bitutil/compress_test.go +++ b/common/bitutil/compress_test.go @@ -117,7 +117,7 @@ func TestDecodingCycle(t *testing.T) { // TestCompression tests that compression works by returning either the bitset // encoded input, or the actual input if the bitset version is longer. func TestCompression(t *testing.T) { - // Check the the compression returns the bitset encoding is shorter + // Check the compression returns the bitset encoding is shorter in := hexutil.MustDecode("0x4912385c0e7b64000000") out := hexutil.MustDecode("0x80fe4912385c0e7b64") @@ -127,7 +127,7 @@ func TestCompression(t *testing.T) { if data, err := DecompressBytes(out, len(in)); err != nil || !bytes.Equal(data, in) { t.Errorf("decoding mismatch for sparse data: have %x, want %x, error %v", data, in, err) } - // Check the the compression returns the input if the bitset encoding is longer + // Check the compression returns the input if the bitset encoding is longer in = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb") out = hexutil.MustDecode("0xdf7070533534333636313639343638373532313536346c1bc33339343837313070706336343035336336346c65fefb3930393233383838ac2f65fefb") diff --git a/console/prompter.go b/console/prompter.go index 9b90034db9..29a53aeadd 100644 --- a/console/prompter.go +++ b/console/prompter.go @@ -43,7 +43,7 @@ type UserPrompter interface { // choice to be made, returning that choice. PromptConfirm(prompt string) (bool, error) - // SetHistory sets the the input scrollback history that the prompter will allow + // SetHistory sets the input scrollback history that the prompter will allow // the user to scroll back to. SetHistory(history []string) @@ -149,7 +149,7 @@ func (p *terminalPrompter) PromptConfirm(prompt string) (bool, error) { return false, err } -// SetHistory sets the the input scrollback history that the prompter will allow +// SetHistory sets the input scrollback history that the prompter will allow // the user to scroll back to. func (p *terminalPrompter) SetHistory(history []string) { p.State.ReadHistory(strings.NewReader(strings.Join(history, "\n"))) diff --git a/core/block_validator.go b/core/block_validator.go index 98958809b7..ecd6a89bc2 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -45,7 +45,7 @@ func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engin return validator } -// ValidateBody validates the given block's uncles and verifies the the block +// ValidateBody validates the given block's uncles and verifies the block // header's transaction and uncle roots. The headers are assumed to be already // validated at this point. func (v *BlockValidator) ValidateBody(block *types.Block) error { diff --git a/core/state_processor.go b/core/state_processor.go index 8e238ce1f0..1a91a57ab4 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -61,7 +61,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg allLogs []*types.Log gp = new(GasPool).AddGas(block.GasLimit()) ) - // Mutate the the block and state according to any hard-fork specs + // Mutate the block and state according to any hard-fork specs if p.config.DAOForkSupport && p.config.DAOForkBlock != nil && p.config.DAOForkBlock.Cmp(block.Number()) == 0 { misc.ApplyDAOHardFork(statedb) } diff --git a/core/vm/contract.go b/core/vm/contract.go index b466681dbd..26bca68951 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -40,7 +40,7 @@ type AccountRef common.Address func (ar AccountRef) Address() common.Address { return (common.Address)(ar) } // Contract represents an ethereum contract in the state database. It contains -// the the contract code, calling arguments. Contract implements ContractRef +// the contract code, calling arguments. Contract implements ContractRef type Contract struct { // CallerAddress is the result of the caller which initialised this // contract. However when the "call method" is delegated this value diff --git a/crypto/secp256k1/secp256.go b/crypto/secp256k1/secp256.go index 843fb12525..35d0eef34a 100644 --- a/crypto/secp256k1/secp256.go +++ b/crypto/secp256k1/secp256.go @@ -86,7 +86,7 @@ func Sign(msg []byte, seckey []byte) ([]byte, error) { return sig, nil } -// RecoverPubkey returns the the public key of the signer. +// RecoverPubkey returns the public key of the signer. // msg must be the 32-byte hash of the message to be signed. // sig must be a 65-byte compact ECDSA signature containing the // recovery id as the last element. diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 984dd13d68..8529535ba7 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -662,7 +662,7 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, for _, header := range request.Headers { taskQueue.Push(header, -float32(header.Number.Uint64())) } - // Add the peer to the expiry report along the the number of failed requests + // Add the peer to the expiry report along the number of failed requests expiries[id] = len(request.Headers) } } diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index 6383596dce..277f14b81a 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -204,7 +204,7 @@ func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time } } -// Enqueue tries to fill gaps the the fetcher's future import queue. +// Enqueue tries to fill gaps the fetcher's future import queue. func (f *Fetcher) Enqueue(peer string, block *types.Block) error { op := &inject{ origin: peer, diff --git a/p2p/server.go b/p2p/server.go index 8f3a511f31..f9a38fcc64 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -77,7 +77,7 @@ type Config struct { // Disabling is useful for protocol debugging (manual topology). NoDiscovery bool - // DiscoveryV5 specifies whether the the new topic-discovery based V5 discovery + // DiscoveryV5 specifies whether the new topic-discovery based V5 discovery // protocol should be started or not. DiscoveryV5 bool `toml:",omitempty"` diff --git a/swarm/api/api.go b/swarm/api/api.go index adf469cfaa..0d08eeea87 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -153,7 +153,7 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // Resolve resolves address by choosing a Resolver by TLD. // If there are more default Resolvers, or for a specific TLD, -// the Hash from the the first one which does not return error +// the Hash from the first one which does not return error // will be returned. func (m *MultiResolver) Resolve(addr string) (h common.Hash, err error) { rs, err := m.getResolveValidator(addr) diff --git a/swarm/network/README.md b/swarm/network/README.md index ad429b38be..e9c9d30d0b 100644 --- a/swarm/network/README.md +++ b/swarm/network/README.md @@ -133,7 +133,7 @@ As part of the deletion protocol then, hashes of insured chunks to be removed ar Downstream peer on the other hand needs to make sure that they can only be finger pointed about a chunk they did receive and store. For this the check of a state should be exhaustive. If historical syncing finishes on one state, all hashes before are covered, no surprises. In other words historical syncing this process is self verifying. With session syncing however, it is not enough to check going back covering the range from old offset to new. Continuity (i.e., that the new state is extension of the old) needs to be verified: after downstream peer reads the range into a buffer, it appends the buffer the last known state at the last known offset and verifies the resulting hash matches -the latest state. Past intervals of historical syncing are checked via the the session root. +the latest state. Past intervals of historical syncing are checked via the session root. Upstream peer signs the states, downstream peers can use as handover proofs. Downstream peers sign off on a state together with an initial offset. diff --git a/swarm/network/stream/intervals/intervals.go b/swarm/network/stream/intervals/intervals.go index 5fd820da87..562c3df9ae 100644 --- a/swarm/network/stream/intervals/intervals.go +++ b/swarm/network/stream/intervals/intervals.go @@ -101,7 +101,7 @@ func (i *Intervals) add(start, end uint64) { } } -// Merge adds all the intervals from the the m Interval to current one. +// Merge adds all the intervals from the m Interval to current one. func (i *Intervals) Merge(m *Intervals) { m.mu.RLock() defer m.mu.RUnlock() diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 6acab50af4..4e1ab09fcf 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -182,7 +182,7 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { conf.addrToIDMap[string(a)] = n } - //get the the node at that index + //get the node at that index //this is the node selected for upload node := sim.RandomUpNode() item, ok := sim.NodeItem(node.ID, bucketKeyStore) diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index 5a2b25c9ec..8e0662327a 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -291,7 +291,7 @@ func (w *Whisper) AddKeyPair(key *ecdsa.PrivateKey) (string, error) { return id, nil } -// HasKeyPair checks if the the whisper node is configured with the private key +// HasKeyPair checks if the whisper node is configured with the private key // of the specified public pair. func (w *Whisper) HasKeyPair(id string) bool { w.keyMu.RLock() diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index 76d485808d..34c1256a6f 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -423,7 +423,7 @@ func (whisper *Whisper) AddKeyPair(key *ecdsa.PrivateKey) (string, error) { return id, nil } -// HasKeyPair checks if the the whisper node is configured with the private key +// HasKeyPair checks if the whisper node is configured with the private key // of the specified public pair. func (whisper *Whisper) HasKeyPair(id string) bool { whisper.keyMu.RLock() From f0488e80f7837094ac74327f0a4f5f6a270641f4 Mon Sep 17 00:00:00 2001 From: Caesar Chad Date: Mon, 27 Aug 2018 16:51:26 +0800 Subject: [PATCH 56/96] signer/storage: fix typo (#17504) --- signer/storage/aes_gcm_storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signer/storage/aes_gcm_storage.go b/signer/storage/aes_gcm_storage.go index 225276667f..399637a443 100644 --- a/signer/storage/aes_gcm_storage.go +++ b/signer/storage/aes_gcm_storage.go @@ -36,7 +36,7 @@ type storedCredential struct { CipherText []byte `json:"c"` } -// AESEncryptedStorage is a storage type which is backed by a json-faile. The json-file contains +// AESEncryptedStorage is a storage type which is backed by a json-file. The json-file contains // key-value mappings, where the keys are _not_ encrypted, only the values are. type AESEncryptedStorage struct { // File to read/write credentials From 1acefafe2201ba061a58b51afd7a7edf4aa0f120 Mon Sep 17 00:00:00 2001 From: Geon Kim Date: Mon, 27 Aug 2018 17:51:58 +0900 Subject: [PATCH 57/96] swarm/api: fix typo (#17500) --- swarm/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 0d08eeea87..d733ad989c 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -111,7 +111,7 @@ func (e *NoResolverError) Error() string { } // MultiResolver is used to resolve URL addresses based on their TLDs. -// Each TLD can have multiple resolvers, and the resoluton from the +// Each TLD can have multiple resolvers, and the resolution from the // first one in the sequence will be returned. type MultiResolver struct { resolvers map[string][]ResolveValidator From f236ac710e075f2d3ef9352a0dc19b3bdf6f5bd8 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Tue, 28 Aug 2018 09:03:33 +0200 Subject: [PATCH 58/96] vendor: github.com/rjeczalik/notify update to master (#17527) --- .../rjeczalik/notify/watcher_fsevents_cgo.go | 6 +++--- .../rjeczalik/notify/watcher_fsevents_go1.10.go | 14 -------------- .../rjeczalik/notify/watcher_fsevents_go1.11.go | 9 --------- vendor/vendor.json | 6 +++--- 4 files changed, 6 insertions(+), 29 deletions(-) delete mode 100644 vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go delete mode 100644 vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go index 95ee704444..cf0416c37e 100644 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go +++ b/vendor/github.com/rjeczalik/notify/watcher_fsevents_cgo.go @@ -48,7 +48,7 @@ var wg sync.WaitGroup // used to wait until the runloop starts // started and is ready via the wg. It also serves purpose of a dummy source, // thanks to it the runloop does not return as it also has at least one source // registered. -var source = C.CFRunLoopSourceCreate(refZero, 0, &C.CFRunLoopSourceContext{ +var source = C.CFRunLoopSourceCreate(C.kCFAllocatorDefault, 0, &C.CFRunLoopSourceContext{ perform: (C.CFRunLoopPerformCallBack)(C.gosource), }) @@ -166,8 +166,8 @@ func (s *stream) Start() error { return nil } wg.Wait() - p := C.CFStringCreateWithCStringNoCopy(refZero, C.CString(s.path), C.kCFStringEncodingUTF8, refZero) - path := C.CFArrayCreate(refZero, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) + p := C.CFStringCreateWithCStringNoCopy(C.kCFAllocatorDefault, C.CString(s.path), C.kCFStringEncodingUTF8, C.kCFAllocatorDefault) + path := C.CFArrayCreate(C.kCFAllocatorDefault, (*unsafe.Pointer)(unsafe.Pointer(&p)), 1, nil) ctx := C.FSEventStreamContext{} ref := C.EventStreamCreate(&ctx, C.uintptr_t(s.info), path, C.FSEventStreamEventId(atomic.LoadUint64(&since)), latency, flags) if ref == nilstream { diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go deleted file mode 100644 index dd2433d20a..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.10.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2018 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,cgo,!go1.11 - -package notify - -/* - #include -*/ -import "C" - -var refZero = (*C.struct___CFAllocator)(nil) diff --git a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go b/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go deleted file mode 100644 index 92b406ce7c..0000000000 --- a/vendor/github.com/rjeczalik/notify/watcher_fsevents_go1.11.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2018 The Notify Authors. All rights reserved. -// Use of this source code is governed by the MIT license that can be -// found in the LICENSE file. - -// +build darwin,!kqueue,go1.11 - -package notify - -const refZero = 0 diff --git a/vendor/vendor.json b/vendor/vendor.json index 0bfc1ca0c1..fea2c804ea 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -370,10 +370,10 @@ "revisionTime": "2017-08-14T17:01:13Z" }, { - "checksumSHA1": "D8AVDI39CJ+jvw0HOotYU2gz54c=", + "checksumSHA1": "lU41NL1TEDtsrr0yUdp3SMB4Y9o=", "path": "github.com/rjeczalik/notify", - "revision": "4e54e7fd043e865c50bda93359fb78813a8d165b", - "revisionTime": "2018-08-08T20:39:25Z" + "revision": "0f065fa99b48b842c3fd3e2c8b194c6f2b69f6b8", + "revisionTime": "2018-08-27T19:31:19Z" }, { "checksumSHA1": "5uqO4ITTDMklKi3uNaE/D9LQ5nM=", From 0bec85f2e27d7b79f7700643cd9b38e74cecadfa Mon Sep 17 00:00:00 2001 From: Sheldon <11510383@mail.sustc.edu.cn> Date: Tue, 28 Aug 2018 15:04:33 +0800 Subject: [PATCH 59/96] core: fix typos in comment (#17531) --- core/tx_journal.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/tx_journal.go b/core/tx_journal.go index 1397e9fd34..41b5156d4a 100644 --- a/core/tx_journal.go +++ b/core/tx_journal.go @@ -34,7 +34,7 @@ var errNoActiveJournal = errors.New("no active journal") // devNull is a WriteCloser that just discards anything written into it. Its // goal is to allow the transaction journal to write into a fake journal when // loading transactions on startup without printing warnings due to no file -// being readt for write. +// being read for write. type devNull struct{} func (*devNull) Write(p []byte) (n int, err error) { return len(p), nil } @@ -57,7 +57,7 @@ func newTxJournal(path string) *txJournal { // load parses a transaction journal dump from disk, loading its contents into // the specified pool. func (journal *txJournal) load(add func([]*types.Transaction) []error) error { - // Skip the parsing if the journal file doens't exist at all + // Skip the parsing if the journal file doesn't exist at all if _, err := os.Stat(journal.path); os.IsNotExist(err) { return nil } @@ -78,7 +78,7 @@ func (journal *txJournal) load(add func([]*types.Transaction) []error) error { // Create a method to load a limited batch of transactions and bump the // appropriate progress counters. Then use this method to load all the - // journalled transactions in small-ish batches. + // journaled transactions in small-ish batches. loadBatch := func(txs types.Transactions) { for _, err := range add(txs) { if err != nil { @@ -103,7 +103,7 @@ func (journal *txJournal) load(add func([]*types.Transaction) []error) error { } break } - // New transaction parsed, queue up for later, import if threnshold is reached + // New transaction parsed, queue up for later, import if threshold is reached total++ if batch = append(batch, tx); batch.Len() > 1024 { From c64d72bea207ccaca3f6aded25d8730a4b8696cd Mon Sep 17 00:00:00 2001 From: Mymskmkt <1847234666@qq.com> Date: Tue, 28 Aug 2018 15:05:25 +0800 Subject: [PATCH 60/96] consensus/ethash: remove unnecessary type declaration (#17529) --- consensus/ethash/consensus.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 259cc5056b..af0e733ae7 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -38,10 +38,10 @@ import ( // Ethash proof-of-work protocol constants. var ( - FrontierBlockReward *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block - ByzantiumBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium - maxUncles = 2 // Maximum number of uncles allowed in a single block - allowedFutureBlockTime = 15 * time.Second // Max time from current time allowed for blocks, before they're considered future blocks + FrontierBlockReward = big.NewInt(5e+18) // Block reward in wei for successfully mining a block + ByzantiumBlockReward = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium + maxUncles = 2 // Maximum number of uncles allowed in a single block + allowedFutureBlockTime = 15 * time.Second // Max time from current time allowed for blocks, before they're considered future blocks ) // Various error messages to mark blocks invalid. These should be private to From b69476b372a26679e5bdb33db3d508f2c955e7ff Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 28 Aug 2018 15:08:16 +0800 Subject: [PATCH 61/96] all: make indexer configurable (#17188) --- core/chain_indexer.go | 1 - eth/backend.go | 4 +- eth/bloombits.go | 15 ++-- les/api_backend.go | 2 +- les/backend.go | 13 +-- les/bloombits.go | 4 +- les/commons.go | 6 +- les/handler.go | 13 +-- les/handler_test.go | 177 ++++++++++++++++++++------------------- les/helper_test.go | 159 +++++++++++++++++++++++++++++++---- les/odr.go | 15 +++- les/odr_requests.go | 20 ++++- les/odr_test.go | 45 ++++------ les/peer.go | 30 +++---- les/request_test.go | 43 +++------- les/server.go | 14 ++-- light/lightchain.go | 20 +++-- light/lightchain_test.go | 9 +- light/odr.go | 5 +- light/odr_test.go | 11 ++- light/odr_util.go | 18 ++-- light/postprocess.go | 174 ++++++++++++++++++++++---------------- light/trie_test.go | 2 +- light/txpool_test.go | 2 +- params/network_params.go | 32 ++++++- 25 files changed, 513 insertions(+), 321 deletions(-) diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 11a7c96fa0..b80b517d91 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -322,7 +322,6 @@ func (c *ChainIndexer) updateLoop() { updating = false c.log.Info("Finished upgrading chain index") } - c.cascadedHead = c.storedSections*c.sectionSize - 1 for _, child := range c.children { c.log.Trace("Cascading chain index update", "head", c.cascadedHead) diff --git a/eth/backend.go b/eth/backend.go index c530d3c7c3..da7e0b2cd5 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -136,7 +136,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { gasPrice: config.MinerGasPrice, etherbase: config.Etherbase, bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, bloomConfirms), + bloomIndexer: NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms), } log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId) @@ -426,7 +426,7 @@ func (s *Ethereum) Protocols() []p2p.Protocol { // Ethereum protocol implementation. func (s *Ethereum) Start(srvr *p2p.Server) error { // Start the bloom bits servicing goroutines - s.startBloomHandlers() + s.startBloomHandlers(params.BloomBitsBlocks) // Start the RPC service s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.NetVersion()) diff --git a/eth/bloombits.go b/eth/bloombits.go index eb18565e2c..c7bb561402 100644 --- a/eth/bloombits.go +++ b/eth/bloombits.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" ) const ( @@ -50,7 +49,7 @@ const ( // startBloomHandlers starts a batch of goroutines to accept bloom bit database // retrievals from possibly a range of filters and serving the data to satisfy. -func (eth *Ethereum) startBloomHandlers() { +func (eth *Ethereum) startBloomHandlers(sectionSize uint64) { for i := 0; i < bloomServiceThreads; i++ { go func() { for { @@ -62,9 +61,9 @@ func (eth *Ethereum) startBloomHandlers() { task := <-request task.Bitsets = make([][]byte, len(task.Sections)) for i, section := range task.Sections { - head := rawdb.ReadCanonicalHash(eth.chainDb, (section+1)*params.BloomBitsBlocks-1) + head := rawdb.ReadCanonicalHash(eth.chainDb, (section+1)*sectionSize-1) if compVector, err := rawdb.ReadBloomBits(eth.chainDb, task.Bit, section, head); err == nil { - if blob, err := bitutil.DecompressBytes(compVector, int(params.BloomBitsBlocks)/8); err == nil { + if blob, err := bitutil.DecompressBytes(compVector, int(sectionSize/8)); err == nil { task.Bitsets[i] = blob } else { task.Error = err @@ -81,10 +80,6 @@ func (eth *Ethereum) startBloomHandlers() { } const ( - // bloomConfirms is the number of confirmation blocks before a bloom section is - // considered probably final and its rotated bits are calculated. - bloomConfirms = 256 - // bloomThrottling is the time to wait between processing two consecutive index // sections. It's useful during chain upgrades to prevent disk overload. bloomThrottling = 100 * time.Millisecond @@ -102,14 +97,14 @@ type BloomIndexer struct { // NewBloomIndexer returns a chain indexer that generates bloom bits data for the // canonical chain for fast logs filtering. -func NewBloomIndexer(db ethdb.Database, size, confReq uint64) *core.ChainIndexer { +func NewBloomIndexer(db ethdb.Database, size, confirms uint64) *core.ChainIndexer { backend := &BloomIndexer{ db: db, size: size, } table := ethdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) - return core.NewChainIndexer(db, table, backend, size, confReq, bloomThrottling, "bloombits") + return core.NewChainIndexer(db, table, backend, size, confirms, bloomThrottling, "bloombits") } // Reset implements core.ChainIndexerBackend, starting a new bloombits index diff --git a/les/api_backend.go b/les/api_backend.go index 4232d3ae07..aa748a4ea1 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -192,7 +192,7 @@ func (b *LesApiBackend) BloomStatus() (uint64, uint64) { return 0, 0 } sections, _, _ := b.eth.bloomIndexer.Sections() - return light.BloomTrieFrequency, sections + return params.BloomBitsBlocksClient, sections } func (b *LesApiBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { diff --git a/les/backend.go b/les/backend.go index 00025ba634..75049da085 100644 --- a/les/backend.go +++ b/les/backend.go @@ -95,6 +95,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { lesCommons: lesCommons{ chainDb: chainDb, config: config, + iConfig: light.DefaultClientIndexerConfig, }, chainConfig: chainConfig, eventMux: ctx.EventMux, @@ -105,16 +106,16 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { shutdownChan: make(chan bool), networkId: config.NetworkId, bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: eth.NewBloomIndexer(chainDb, light.BloomTrieFrequency, light.HelperTrieConfirmations), + bloomIndexer: eth.NewBloomIndexer(chainDb, params.BloomBitsBlocksClient, params.HelperTrieConfirmations), } leth.relay = NewLesTxRelay(peers, leth.reqDist) leth.serverPool = newServerPool(chainDb, quitSync, &leth.wg) leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool) - leth.odr = NewLesOdr(chainDb, leth.retriever) - leth.chtIndexer = light.NewChtIndexer(chainDb, true, leth.odr) - leth.bloomTrieIndexer = light.NewBloomTrieIndexer(chainDb, true, leth.odr) + leth.odr = NewLesOdr(chainDb, light.DefaultClientIndexerConfig, leth.retriever) + leth.chtIndexer = light.NewChtIndexer(chainDb, leth.odr, params.CHTFrequencyClient, params.HelperTrieConfirmations) + leth.bloomTrieIndexer = light.NewBloomTrieIndexer(chainDb, leth.odr, params.BloomBitsBlocksClient, params.BloomTrieFrequency) leth.odr.SetIndexers(leth.chtIndexer, leth.bloomTrieIndexer, leth.bloomIndexer) // Note: NewLightChain adds the trusted checkpoint so it needs an ODR with @@ -135,7 +136,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { } leth.txPool = light.NewTxPool(leth.chainConfig, leth.blockchain, leth.relay) - if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, true, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, leth.serverPool, quitSync, &leth.wg); err != nil { + if leth.protocolManager, err = NewProtocolManager(leth.chainConfig, light.DefaultClientIndexerConfig, true, config.NetworkId, leth.eventMux, leth.engine, leth.peers, leth.blockchain, nil, chainDb, leth.odr, leth.relay, leth.serverPool, quitSync, &leth.wg); err != nil { return nil, err } leth.ApiBackend = &LesApiBackend{leth, nil} @@ -230,8 +231,8 @@ func (s *LightEthereum) Protocols() []p2p.Protocol { // Start implements node.Service, starting all internal goroutines needed by the // Ethereum protocol implementation. func (s *LightEthereum) Start(srvr *p2p.Server) error { - s.startBloomHandlers() log.Warn("Light client mode is an experimental feature") + s.startBloomHandlers(params.BloomBitsBlocksClient) s.netRPCService = ethapi.NewPublicNetAPI(srvr, s.networkId) // clients are searching for the first advertised protocol in the list protocolVersion := AdvertiseProtocolVersions[0] diff --git a/les/bloombits.go b/les/bloombits.go index 2871a90064..aea0fcd5f4 100644 --- a/les/bloombits.go +++ b/les/bloombits.go @@ -43,7 +43,7 @@ const ( // startBloomHandlers starts a batch of goroutines to accept bloom bit database // retrievals from possibly a range of filters and serving the data to satisfy. -func (eth *LightEthereum) startBloomHandlers() { +func (eth *LightEthereum) startBloomHandlers(sectionSize uint64) { for i := 0; i < bloomServiceThreads; i++ { go func() { for { @@ -57,7 +57,7 @@ func (eth *LightEthereum) startBloomHandlers() { compVectors, err := light.GetBloomBits(task.Context, eth.odr, task.Bit, task.Sections) if err == nil { for i := range task.Sections { - if blob, err := bitutil.DecompressBytes(compVectors[i], int(light.BloomTrieFrequency/8)); err == nil { + if blob, err := bitutil.DecompressBytes(compVectors[i], int(sectionSize/8)); err == nil { task.Bitsets[i] = blob } else { task.Error = err diff --git a/les/commons.go b/les/commons.go index d8e9412951..a97687993d 100644 --- a/les/commons.go +++ b/les/commons.go @@ -33,6 +33,7 @@ import ( // lesCommons contains fields needed by both server and client. type lesCommons struct { config *eth.Config + iConfig *light.IndexerConfig chainDb ethdb.Database protocolManager *ProtocolManager chtIndexer, bloomTrieIndexer *core.ChainIndexer @@ -81,7 +82,7 @@ func (c *lesCommons) nodeInfo() interface{} { if !c.protocolManager.lightSync { // convert to client section size if running in server mode - sections /= light.CHTFrequencyClient / light.CHTFrequencyServer + sections /= c.iConfig.PairChtSize / c.iConfig.ChtSize } if sections2 < sections { @@ -94,7 +95,8 @@ func (c *lesCommons) nodeInfo() interface{} { if c.protocolManager.lightSync { chtRoot = light.GetChtRoot(c.chainDb, sectionIndex, sectionHead) } else { - chtRoot = light.GetChtV2Root(c.chainDb, sectionIndex, sectionHead) + idxV2 := (sectionIndex+1)*c.iConfig.PairChtSize/c.iConfig.ChtSize - 1 + chtRoot = light.GetChtRoot(c.chainDb, idxV2, sectionHead) } cht = light.TrustedCheckpoint{ SectionIdx: sectionIndex, diff --git a/les/handler.go b/les/handler.go index ca40eaabfa..243a6dabd4 100644 --- a/les/handler.go +++ b/les/handler.go @@ -94,6 +94,7 @@ type ProtocolManager struct { txrelay *LesTxRelay networkId uint64 chainConfig *params.ChainConfig + iConfig *light.IndexerConfig blockchain BlockChain chainDb ethdb.Database odr *LesOdr @@ -123,13 +124,14 @@ type ProtocolManager struct { // NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the ethereum network. -func NewProtocolManager(chainConfig *params.ChainConfig, lightSync bool, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, serverPool *serverPool, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { +func NewProtocolManager(chainConfig *params.ChainConfig, indexerConfig *light.IndexerConfig, lightSync bool, networkId uint64, mux *event.TypeMux, engine consensus.Engine, peers *peerSet, blockchain BlockChain, txpool txPool, chainDb ethdb.Database, odr *LesOdr, txrelay *LesTxRelay, serverPool *serverPool, quitSync chan struct{}, wg *sync.WaitGroup) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ lightSync: lightSync, eventMux: mux, blockchain: blockchain, chainConfig: chainConfig, + iConfig: indexerConfig, chainDb: chainDb, odr: odr, networkId: networkId, @@ -882,7 +884,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) for _, req := range req.Reqs { if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { - sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*light.CHTFrequencyServer-1) + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*pm.iConfig.ChtSize-1) if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { trie, err := trie.New(root, trieDb) if err != nil { @@ -1137,10 +1139,11 @@ func (pm *ProtocolManager) getAccount(statedb *state.StateDB, root, hash common. func (pm *ProtocolManager) getHelperTrie(id uint, idx uint64) (common.Hash, string) { switch id { case htCanonical: - sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idx+1)*light.CHTFrequencyClient-1) - return light.GetChtV2Root(pm.chainDb, idx, sectionHead), light.ChtTablePrefix + idxV1 := (idx+1)*(pm.iConfig.PairChtSize/pm.iConfig.ChtSize) - 1 + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idxV1+1)*pm.iConfig.ChtSize-1) + return light.GetChtRoot(pm.chainDb, idxV1, sectionHead), light.ChtTablePrefix case htBloomBits: - sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idx+1)*light.BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, (idx+1)*pm.iConfig.BloomTrieSize-1) return light.GetBloomTrieRoot(pm.chainDb, idx, sectionHead), light.BloomTrieTablePrefix } return common.Hash{}, "" diff --git a/les/handler_test.go b/les/handler_test.go index 31aad3ed45..43be7f41b1 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -51,10 +51,9 @@ func TestGetBlockHeadersLes1(t *testing.T) { testGetBlockHeaders(t, 1) } func TestGetBlockHeadersLes2(t *testing.T) { testGetBlockHeaders(t, 2) } func testGetBlockHeaders(t *testing.T, protocol int) { - pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil, nil, ethdb.NewMemDatabase()) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() + server, tearDown := newServerEnv(t, downloader.MaxHashFetch+15, protocol, nil) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) // Create a "random" unknown hash for testing var unknown common.Hash @@ -167,9 +166,9 @@ func testGetBlockHeaders(t *testing.T, protocol int) { } // Send the hash request and verify the response reqID++ - cost := peer.GetRequestCost(GetBlockHeadersMsg, int(tt.query.Amount)) - sendRequest(peer.app, GetBlockHeadersMsg, reqID, cost, tt.query) - if err := expectResponse(peer.app, BlockHeadersMsg, reqID, testBufLimit, headers); err != nil { + cost := server.tPeer.GetRequestCost(GetBlockHeadersMsg, int(tt.query.Amount)) + sendRequest(server.tPeer.app, GetBlockHeadersMsg, reqID, cost, tt.query) + if err := expectResponse(server.tPeer.app, BlockHeadersMsg, reqID, testBufLimit, headers); err != nil { t.Errorf("test %d: headers mismatch: %v", i, err) } } @@ -180,10 +179,9 @@ func TestGetBlockBodiesLes1(t *testing.T) { testGetBlockBodies(t, 1) } func TestGetBlockBodiesLes2(t *testing.T) { testGetBlockBodies(t, 2) } func testGetBlockBodies(t *testing.T, protocol int) { - pm := newTestProtocolManagerMust(t, false, downloader.MaxBlockFetch+15, nil, nil, nil, ethdb.NewMemDatabase()) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() + server, tearDown := newServerEnv(t, downloader.MaxBlockFetch+15, protocol, nil) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) // Create a batch of tests for various scenarios limit := MaxBodyFetch @@ -243,9 +241,9 @@ func testGetBlockBodies(t *testing.T, protocol int) { } reqID++ // Send the hash request and verify the response - cost := peer.GetRequestCost(GetBlockBodiesMsg, len(hashes)) - sendRequest(peer.app, GetBlockBodiesMsg, reqID, cost, hashes) - if err := expectResponse(peer.app, BlockBodiesMsg, reqID, testBufLimit, bodies); err != nil { + cost := server.tPeer.GetRequestCost(GetBlockBodiesMsg, len(hashes)) + sendRequest(server.tPeer.app, GetBlockBodiesMsg, reqID, cost, hashes) + if err := expectResponse(server.tPeer.app, BlockBodiesMsg, reqID, testBufLimit, bodies); err != nil { t.Errorf("test %d: bodies mismatch: %v", i, err) } } @@ -257,10 +255,9 @@ func TestGetCodeLes2(t *testing.T) { testGetCode(t, 2) } func testGetCode(t *testing.T, protocol int) { // Assemble the test environment - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, ethdb.NewMemDatabase()) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() + server, tearDown := newServerEnv(t, 4, protocol, nil) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) var codereqs []*CodeReq var codes [][]byte @@ -277,9 +274,9 @@ func testGetCode(t *testing.T, protocol int) { } } - cost := peer.GetRequestCost(GetCodeMsg, len(codereqs)) - sendRequest(peer.app, GetCodeMsg, 42, cost, codereqs) - if err := expectResponse(peer.app, CodeMsg, 42, testBufLimit, codes); err != nil { + cost := server.tPeer.GetRequestCost(GetCodeMsg, len(codereqs)) + sendRequest(server.tPeer.app, GetCodeMsg, 42, cost, codereqs) + if err := expectResponse(server.tPeer.app, CodeMsg, 42, testBufLimit, codes); err != nil { t.Errorf("codes mismatch: %v", err) } } @@ -290,11 +287,9 @@ func TestGetReceiptLes2(t *testing.T) { testGetReceipt(t, 2) } func testGetReceipt(t *testing.T, protocol int) { // Assemble the test environment - db := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() + server, tearDown := newServerEnv(t, 4, protocol, nil) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) // Collect the hashes to request, and the response to expect hashes, receipts := []common.Hash{}, []types.Receipts{} @@ -302,12 +297,12 @@ func testGetReceipt(t *testing.T, protocol int) { block := bc.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) - receipts = append(receipts, rawdb.ReadReceipts(db, block.Hash(), block.NumberU64())) + receipts = append(receipts, rawdb.ReadReceipts(server.db, block.Hash(), block.NumberU64())) } // Send the hash request and verify the response - cost := peer.GetRequestCost(GetReceiptsMsg, len(hashes)) - sendRequest(peer.app, GetReceiptsMsg, 42, cost, hashes) - if err := expectResponse(peer.app, ReceiptsMsg, 42, testBufLimit, receipts); err != nil { + cost := server.tPeer.GetRequestCost(GetReceiptsMsg, len(hashes)) + sendRequest(server.tPeer.app, GetReceiptsMsg, 42, cost, hashes) + if err := expectResponse(server.tPeer.app, ReceiptsMsg, 42, testBufLimit, receipts); err != nil { t.Errorf("receipts mismatch: %v", err) } } @@ -318,11 +313,9 @@ func TestGetProofsLes2(t *testing.T) { testGetProofs(t, 2) } func testGetProofs(t *testing.T, protocol int) { // Assemble the test environment - db := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() + server, tearDown := newServerEnv(t, 4, protocol, nil) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) var ( proofreqs []ProofReq @@ -334,7 +327,7 @@ func testGetProofs(t *testing.T, protocol int) { for i := uint64(0); i <= bc.CurrentBlock().NumberU64(); i++ { header := bc.GetHeaderByNumber(i) root := header.Root - trie, _ := trie.New(root, trie.NewDatabase(db)) + trie, _ := trie.New(root, trie.NewDatabase(server.db)) for _, acc := range accounts { req := ProofReq{ @@ -356,15 +349,15 @@ func testGetProofs(t *testing.T, protocol int) { // Send the proof request and verify the response switch protocol { case 1: - cost := peer.GetRequestCost(GetProofsV1Msg, len(proofreqs)) - sendRequest(peer.app, GetProofsV1Msg, 42, cost, proofreqs) - if err := expectResponse(peer.app, ProofsV1Msg, 42, testBufLimit, proofsV1); err != nil { + cost := server.tPeer.GetRequestCost(GetProofsV1Msg, len(proofreqs)) + sendRequest(server.tPeer.app, GetProofsV1Msg, 42, cost, proofreqs) + if err := expectResponse(server.tPeer.app, ProofsV1Msg, 42, testBufLimit, proofsV1); err != nil { t.Errorf("proofs mismatch: %v", err) } case 2: - cost := peer.GetRequestCost(GetProofsV2Msg, len(proofreqs)) - sendRequest(peer.app, GetProofsV2Msg, 42, cost, proofreqs) - if err := expectResponse(peer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil { + cost := server.tPeer.GetRequestCost(GetProofsV2Msg, len(proofreqs)) + sendRequest(server.tPeer.app, GetProofsV2Msg, 42, cost, proofreqs) + if err := expectResponse(server.tPeer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil { t.Errorf("proofs mismatch: %v", err) } } @@ -375,28 +368,33 @@ func TestGetCHTProofsLes1(t *testing.T) { testGetCHTProofs(t, 1) } func TestGetCHTProofsLes2(t *testing.T) { testGetCHTProofs(t, 2) } func testGetCHTProofs(t *testing.T, protocol int) { - // Figure out the client's CHT frequency - frequency := uint64(light.CHTFrequencyClient) - if protocol == 1 { - frequency = uint64(light.CHTFrequencyServer) + config := light.TestServerIndexerConfig + frequency := config.ChtSize + if protocol == 2 { + frequency = config.PairChtSize } - // Assemble the test environment - db := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, int(frequency)+light.HelperTrieProcessConfirmations, testChainGen, nil, nil, db) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", protocol, pm, true) - defer peer.close() - // Wait a while for the CHT indexer to process the new headers - time.Sleep(100 * time.Millisecond * time.Duration(frequency/light.CHTFrequencyServer)) // Chain indexer throttling - time.Sleep(250 * time.Millisecond) // CI tester slack + waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { + expectSections := frequency / config.ChtSize + for { + cs, _, _ := cIndexer.Sections() + bs, _, _ := bIndexer.Sections() + if cs >= expectSections && bs >= expectSections { + break + } + time.Sleep(10 * time.Millisecond) + } + } + server, tearDown := newServerEnv(t, int(frequency+config.ChtConfirms), protocol, waitIndexers) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) // Assemble the proofs from the different protocols - header := bc.GetHeaderByNumber(frequency) + header := bc.GetHeaderByNumber(frequency - 1) rlp, _ := rlp.EncodeToBytes(header) key := make([]byte, 8) - binary.BigEndian.PutUint64(key, frequency) + binary.BigEndian.PutUint64(key, frequency-1) proofsV1 := []ChtResp{{ Header: header, @@ -406,41 +404,41 @@ func testGetCHTProofs(t *testing.T, protocol int) { } switch protocol { case 1: - root := light.GetChtRoot(db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix))) + root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(server.db, light.ChtTablePrefix))) var proof light.NodeList trie.Prove(key, 0, &proof) proofsV1[0].Proof = proof case 2: - root := light.GetChtV2Root(db, 0, bc.GetHeaderByNumber(frequency-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.ChtTablePrefix))) + root := light.GetChtRoot(server.db, (frequency/config.ChtSize)-1, bc.GetHeaderByNumber(frequency-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(server.db, light.ChtTablePrefix))) trie.Prove(key, 0, &proofsV2.Proofs) } // Assemble the requests for the different protocols requestsV1 := []ChtReq{{ - ChtNum: 1, - BlockNum: frequency, + ChtNum: frequency / config.ChtSize, + BlockNum: frequency - 1, }} requestsV2 := []HelperTrieReq{{ Type: htCanonical, - TrieIdx: 0, + TrieIdx: frequency/config.PairChtSize - 1, Key: key, AuxReq: auxHeader, }} // Send the proof request and verify the response switch protocol { case 1: - cost := peer.GetRequestCost(GetHeaderProofsMsg, len(requestsV1)) - sendRequest(peer.app, GetHeaderProofsMsg, 42, cost, requestsV1) - if err := expectResponse(peer.app, HeaderProofsMsg, 42, testBufLimit, proofsV1); err != nil { + cost := server.tPeer.GetRequestCost(GetHeaderProofsMsg, len(requestsV1)) + sendRequest(server.tPeer.app, GetHeaderProofsMsg, 42, cost, requestsV1) + if err := expectResponse(server.tPeer.app, HeaderProofsMsg, 42, testBufLimit, proofsV1); err != nil { t.Errorf("proofs mismatch: %v", err) } case 2: - cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requestsV2)) - sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requestsV2) - if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil { + cost := server.tPeer.GetRequestCost(GetHelperTrieProofsMsg, len(requestsV2)) + sendRequest(server.tPeer.app, GetHelperTrieProofsMsg, 42, cost, requestsV2) + if err := expectResponse(server.tPeer.app, HelperTrieProofsMsg, 42, testBufLimit, proofsV2); err != nil { t.Errorf("proofs mismatch: %v", err) } } @@ -448,24 +446,31 @@ func testGetCHTProofs(t *testing.T, protocol int) { // Tests that bloombits proofs can be correctly retrieved. func TestGetBloombitsProofs(t *testing.T) { - // Assemble the test environment - db := ethdb.NewMemDatabase() - pm := newTestProtocolManagerMust(t, false, light.BloomTrieFrequency+256, testChainGen, nil, nil, db) - bc := pm.blockchain.(*core.BlockChain) - peer, _ := newTestPeer(t, "peer", 2, pm, true) - defer peer.close() + config := light.TestServerIndexerConfig - // Wait a while for the bloombits indexer to process the new headers - time.Sleep(100 * time.Millisecond * time.Duration(light.BloomTrieFrequency/4096)) // Chain indexer throttling - time.Sleep(250 * time.Millisecond) // CI tester slack + waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { + for { + cs, _, _ := cIndexer.Sections() + bs, _, _ := bIndexer.Sections() + bts, _, _ := btIndexer.Sections() + if cs >= 8 && bs >= 8 && bts >= 1 { + break + } + time.Sleep(10 * time.Millisecond) + } + } + server, tearDown := newServerEnv(t, int(config.BloomTrieSize+config.BloomTrieConfirms), 2, waitIndexers) + defer tearDown() + bc := server.pm.blockchain.(*core.BlockChain) // Request and verify each bit of the bloom bits proofs for bit := 0; bit < 2048; bit++ { - // Assemble therequest and proofs for the bloombits + // Assemble the request and proofs for the bloombits key := make([]byte, 10) binary.BigEndian.PutUint16(key[:2], uint16(bit)) - binary.BigEndian.PutUint64(key[2:], uint64(light.BloomTrieFrequency)) + // Only the first bloom section has data. + binary.BigEndian.PutUint64(key[2:], 0) requests := []HelperTrieReq{{ Type: htBloomBits, @@ -474,14 +479,14 @@ func TestGetBloombitsProofs(t *testing.T) { }} var proofs HelperTrieResps - root := light.GetBloomTrieRoot(db, 0, bc.GetHeaderByNumber(light.BloomTrieFrequency-1).Hash()) - trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(db, light.BloomTrieTablePrefix))) + root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) + trie, _ := trie.New(root, trie.NewDatabase(ethdb.NewTable(server.db, light.BloomTrieTablePrefix))) trie.Prove(key, 0, &proofs.Proofs) // Send the proof request and verify the response - cost := peer.GetRequestCost(GetHelperTrieProofsMsg, len(requests)) - sendRequest(peer.app, GetHelperTrieProofsMsg, 42, cost, requests) - if err := expectResponse(peer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil { + cost := server.tPeer.GetRequestCost(GetHelperTrieProofsMsg, len(requests)) + sendRequest(server.tPeer.app, GetHelperTrieProofsMsg, 42, cost, requests) + if err := expectResponse(server.tPeer.app, HelperTrieProofsMsg, 42, testBufLimit, proofs); err != nil { t.Errorf("bit %d: proofs mismatch: %v", bit, err) } } diff --git a/les/helper_test.go b/les/helper_test.go index 8817c20c7d..206ee2d920 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -24,6 +24,7 @@ import ( "math/big" "sync" "testing" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" @@ -123,6 +124,15 @@ func testChainGen(i int, block *core.BlockGen) { } } +// testIndexers creates a set of indexers with specified params for testing purpose. +func testIndexers(db ethdb.Database, odr light.OdrBackend, iConfig *light.IndexerConfig) (*core.ChainIndexer, *core.ChainIndexer, *core.ChainIndexer) { + chtIndexer := light.NewChtIndexer(db, odr, iConfig.ChtSize, iConfig.ChtConfirms) + bloomIndexer := eth.NewBloomIndexer(db, iConfig.BloomSize, iConfig.BloomConfirms) + bloomTrieIndexer := light.NewBloomTrieIndexer(db, odr, iConfig.BloomSize, iConfig.BloomTrieSize) + bloomIndexer.AddChildIndexer(bloomTrieIndexer) + return chtIndexer, bloomIndexer, bloomTrieIndexer +} + func testRCL() RequestCostList { cl := make(RequestCostList, len(reqList)) for i, code := range reqList { @@ -134,9 +144,9 @@ func testRCL() RequestCostList { } // newTestProtocolManager creates a new protocol manager for testing purposes, -// with the given number of blocks already known, and potential notification -// channels for different events. -func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *core.BlockGen), peers *peerSet, odr *LesOdr, db ethdb.Database) (*ProtocolManager, error) { +// with the given number of blocks already known, potential notification +// channels for different events and relative chain indexers array. +func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *core.BlockGen), odr *LesOdr, peers *peerSet, db ethdb.Database) (*ProtocolManager, error) { var ( evmux = new(event.TypeMux) engine = ethash.NewFaker() @@ -155,16 +165,6 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor chain, _ = light.NewLightChain(odr, gspec.Config, engine) } else { blockchain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) - - chtIndexer := light.NewChtIndexer(db, false, nil) - chtIndexer.Start(blockchain) - - bbtIndexer := light.NewBloomTrieIndexer(db, false, nil) - - bloomIndexer := eth.NewBloomIndexer(db, params.BloomBitsBlocks, light.HelperTrieProcessConfirmations) - bloomIndexer.AddChildIndexer(bbtIndexer) - bloomIndexer.Start(blockchain) - gchain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, blocks, generator) if _, err := blockchain.InsertChain(gchain); err != nil { panic(err) @@ -172,7 +172,11 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor chain = blockchain } - pm, err := NewProtocolManager(gspec.Config, lightSync, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup)) + indexConfig := light.TestServerIndexerConfig + if lightSync { + indexConfig = light.TestClientIndexerConfig + } + pm, err := NewProtocolManager(gspec.Config, indexConfig, lightSync, NetworkId, evmux, engine, peers, chain, nil, db, odr, nil, nil, make(chan struct{}), new(sync.WaitGroup)) if err != nil { return nil, err } @@ -193,11 +197,11 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor } // newTestProtocolManagerMust creates a new protocol manager for testing purposes, -// with the given number of blocks already known, and potential notification -// channels for different events. In case of an error, the constructor force- +// with the given number of blocks already known, potential notification +// channels for different events and relative chain indexers array. In case of an error, the constructor force- // fails the test. -func newTestProtocolManagerMust(t *testing.T, lightSync bool, blocks int, generator func(int, *core.BlockGen), peers *peerSet, odr *LesOdr, db ethdb.Database) *ProtocolManager { - pm, err := newTestProtocolManager(lightSync, blocks, generator, peers, odr, db) +func newTestProtocolManagerMust(t *testing.T, lightSync bool, blocks int, generator func(int, *core.BlockGen), odr *LesOdr, peers *peerSet, db ethdb.Database) *ProtocolManager { + pm, err := newTestProtocolManager(lightSync, blocks, generator, odr, peers, db) if err != nil { t.Fatalf("Failed to create protocol manager: %v", err) } @@ -320,3 +324,122 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNu func (p *testPeer) close() { p.app.Close() } + +// TestEntity represents a network entity for testing with necessary auxiliary fields. +type TestEntity struct { + db ethdb.Database + rPeer *peer + tPeer *testPeer + peers *peerSet + pm *ProtocolManager + // Indexers + chtIndexer *core.ChainIndexer + bloomIndexer *core.ChainIndexer + bloomTrieIndexer *core.ChainIndexer +} + +// newServerEnv creates a server testing environment with a connected test peer for testing purpose. +func newServerEnv(t *testing.T, blocks int, protocol int, waitIndexers func(*core.ChainIndexer, *core.ChainIndexer, *core.ChainIndexer)) (*TestEntity, func()) { + db := ethdb.NewMemDatabase() + cIndexer, bIndexer, btIndexer := testIndexers(db, nil, light.TestServerIndexerConfig) + + pm := newTestProtocolManagerMust(t, false, blocks, testChainGen, nil, nil, db) + peer, _ := newTestPeer(t, "peer", protocol, pm, true) + + cIndexer.Start(pm.blockchain.(*core.BlockChain)) + bIndexer.Start(pm.blockchain.(*core.BlockChain)) + + // Wait until indexers generate enough index data. + if waitIndexers != nil { + waitIndexers(cIndexer, bIndexer, btIndexer) + } + + return &TestEntity{ + db: db, + tPeer: peer, + pm: pm, + chtIndexer: cIndexer, + bloomIndexer: bIndexer, + bloomTrieIndexer: btIndexer, + }, func() { + peer.close() + // Note bloom trie indexer will be closed by it parent recursively. + cIndexer.Close() + bIndexer.Close() + } +} + +// newClientServerEnv creates a client/server arch environment with a connected les server and light client pair +// for testing purpose. +func newClientServerEnv(t *testing.T, blocks int, protocol int, waitIndexers func(*core.ChainIndexer, *core.ChainIndexer, *core.ChainIndexer), newPeer bool) (*TestEntity, *TestEntity, func()) { + db, ldb := ethdb.NewMemDatabase(), ethdb.NewMemDatabase() + peers, lPeers := newPeerSet(), newPeerSet() + + dist := newRequestDistributor(lPeers, make(chan struct{})) + rm := newRetrieveManager(lPeers, dist, nil) + odr := NewLesOdr(ldb, light.TestClientIndexerConfig, rm) + + cIndexer, bIndexer, btIndexer := testIndexers(db, nil, light.TestServerIndexerConfig) + lcIndexer, lbIndexer, lbtIndexer := testIndexers(ldb, odr, light.TestClientIndexerConfig) + odr.SetIndexers(lcIndexer, lbtIndexer, lbIndexer) + + pm := newTestProtocolManagerMust(t, false, blocks, testChainGen, nil, peers, db) + lpm := newTestProtocolManagerMust(t, true, 0, nil, odr, lPeers, ldb) + + startIndexers := func(clientMode bool, pm *ProtocolManager) { + if clientMode { + lcIndexer.Start(pm.blockchain.(*light.LightChain)) + lbIndexer.Start(pm.blockchain.(*light.LightChain)) + } else { + cIndexer.Start(pm.blockchain.(*core.BlockChain)) + bIndexer.Start(pm.blockchain.(*core.BlockChain)) + } + } + + startIndexers(false, pm) + startIndexers(true, lpm) + + // Execute wait until function if it is specified. + if waitIndexers != nil { + waitIndexers(cIndexer, bIndexer, btIndexer) + } + + var ( + peer, lPeer *peer + err1, err2 <-chan error + ) + if newPeer { + peer, err1, lPeer, err2 = newTestPeerPair("peer", protocol, pm, lpm) + select { + case <-time.After(time.Millisecond * 100): + case err := <-err1: + t.Fatalf("peer 1 handshake error: %v", err) + case err := <-err2: + t.Fatalf("peer 2 handshake error: %v", err) + } + } + + return &TestEntity{ + db: db, + pm: pm, + rPeer: peer, + peers: peers, + chtIndexer: cIndexer, + bloomIndexer: bIndexer, + bloomTrieIndexer: btIndexer, + }, &TestEntity{ + db: ldb, + pm: lpm, + rPeer: lPeer, + peers: lPeers, + chtIndexer: lcIndexer, + bloomIndexer: lbIndexer, + bloomTrieIndexer: lbtIndexer, + }, func() { + // Note bloom trie indexers will be closed by their parents recursively. + cIndexer.Close() + bIndexer.Close() + lcIndexer.Close() + lbIndexer.Close() + } +} diff --git a/les/odr.go b/les/odr.go index 2ad28d5d9f..9def05a676 100644 --- a/les/odr.go +++ b/les/odr.go @@ -28,16 +28,18 @@ import ( // LesOdr implements light.OdrBackend type LesOdr struct { db ethdb.Database + indexerConfig *light.IndexerConfig chtIndexer, bloomTrieIndexer, bloomIndexer *core.ChainIndexer retriever *retrieveManager stop chan struct{} } -func NewLesOdr(db ethdb.Database, retriever *retrieveManager) *LesOdr { +func NewLesOdr(db ethdb.Database, config *light.IndexerConfig, retriever *retrieveManager) *LesOdr { return &LesOdr{ - db: db, - retriever: retriever, - stop: make(chan struct{}), + db: db, + indexerConfig: config, + retriever: retriever, + stop: make(chan struct{}), } } @@ -73,6 +75,11 @@ func (odr *LesOdr) BloomIndexer() *core.ChainIndexer { return odr.bloomIndexer } +// IndexerConfig returns the indexer config. +func (odr *LesOdr) IndexerConfig() *light.IndexerConfig { + return odr.indexerConfig +} + const ( MsgBlockBodies = iota MsgCode diff --git a/les/odr_requests.go b/les/odr_requests.go index 075fcd92ca..9e9b2673f9 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -365,7 +365,7 @@ func (r *ChtRequest) CanSend(peer *peer) bool { peer.lock.RLock() defer peer.lock.RUnlock() - return peer.headInfo.Number >= light.HelperTrieConfirmations && r.ChtNum <= (peer.headInfo.Number-light.HelperTrieConfirmations)/light.CHTFrequencyClient + return peer.headInfo.Number >= r.Config.ChtConfirms && r.ChtNum <= (peer.headInfo.Number-r.Config.ChtConfirms)/r.Config.ChtSize } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) @@ -379,7 +379,21 @@ func (r *ChtRequest) Request(reqID uint64, peer *peer) error { Key: encNum[:], AuxReq: auxHeader, } - return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), []HelperTrieReq{req}) + switch peer.version { + case lpv1: + var reqsV1 ChtReq + if req.Type != htCanonical || req.AuxReq != auxHeader || len(req.Key) != 8 { + return fmt.Errorf("Request invalid in LES/1 mode") + } + blockNum := binary.BigEndian.Uint64(req.Key) + // convert HelperTrie request to old CHT request + reqsV1 = ChtReq{ChtNum: (req.TrieIdx + 1) * (r.Config.ChtSize / r.Config.PairChtSize), BlockNum: blockNum, FromLevel: req.FromLevel} + return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), []ChtReq{reqsV1}) + case lpv2: + return peer.RequestHelperTrieProofs(reqID, r.GetCost(peer), []HelperTrieReq{req}) + default: + panic(nil) + } } // Valid processes an ODR request reply message from the LES network @@ -484,7 +498,7 @@ func (r *BloomRequest) CanSend(peer *peer) bool { if peer.version < lpv2 { return false } - return peer.headInfo.Number >= light.HelperTrieConfirmations && r.BloomTrieNum <= (peer.headInfo.Number-light.HelperTrieConfirmations)/light.BloomTrieFrequency + return peer.headInfo.Number >= r.Config.BloomTrieConfirms && r.BloomTrieNum <= (peer.headInfo.Number-r.Config.BloomTrieConfirms)/r.Config.BloomTrieSize } // Request sends an ODR request to the LES network (implementation of LesOdrRequest) diff --git a/les/odr_test.go b/les/odr_test.go index c7c25cbe4b..e6458adf56 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/params" @@ -160,36 +159,21 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai return res } +// testOdr tests odr requests whose validation guaranteed by block headers. func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { // Assemble the test environment - peers := newPeerSet() - dist := newRequestDistributor(peers, make(chan struct{})) - rm := newRetrieveManager(peers, dist, nil) - db := ethdb.NewMemDatabase() - ldb := ethdb.NewMemDatabase() - odr := NewLesOdr(ldb, rm) - odr.SetIndexers(light.NewChtIndexer(db, true, nil), light.NewBloomTrieIndexer(db, true, nil), eth.NewBloomIndexer(db, light.BloomTrieFrequency, light.HelperTrieConfirmations)) - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) - lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) - _, err1, lpeer, err2 := newTestPeerPair("peer", protocol, pm, lpm) - select { - case <-time.After(time.Millisecond * 100): - case err := <-err1: - t.Fatalf("peer 1 handshake error: %v", err) - case err := <-err2: - t.Fatalf("peer 1 handshake error: %v", err) - } - - lpm.synchronise(lpeer) + server, client, tearDown := newClientServerEnv(t, 4, protocol, nil, true) + defer tearDown() + client.pm.synchronise(client.rPeer) test := func(expFail uint64) { - for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := rawdb.ReadCanonicalHash(db, i) - b1 := fn(light.NoOdr, db, pm.chainConfig, pm.blockchain.(*core.BlockChain), nil, bhash) + for i := uint64(0); i <= server.pm.blockchain.CurrentHeader().Number.Uint64(); i++ { + bhash := rawdb.ReadCanonicalHash(server.db, i) + b1 := fn(light.NoOdr, server.db, server.pm.chainConfig, server.pm.blockchain.(*core.BlockChain), nil, bhash) ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() - b2 := fn(ctx, ldb, lpm.chainConfig, nil, lpm.blockchain.(*light.LightChain), bhash) + b2 := fn(ctx, client.db, client.pm.chainConfig, nil, client.pm.blockchain.(*light.LightChain), bhash) eq := bytes.Equal(b1, b2) exp := i < expFail @@ -201,21 +185,20 @@ func testOdr(t *testing.T, protocol int, expFail uint64, fn odrTestFn) { } } } - // temporarily remove peer to test odr fails // expect retrievals to fail (except genesis block) without a les peer - peers.Unregister(lpeer.id) + client.peers.Unregister(client.rPeer.id) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed test(expFail) // expect all retrievals to pass - peers.Register(lpeer) + client.peers.Register(client.rPeer) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed - lpeer.lock.Lock() - lpeer.hasBlock = func(common.Hash, uint64) bool { return true } - lpeer.lock.Unlock() + client.peers.lock.Lock() + client.rPeer.hasBlock = func(common.Hash, uint64) bool { return true } + client.peers.lock.Unlock() test(5) // still expect all retrievals to pass, now data should be cached locally - peers.Unregister(lpeer.id) + client.peers.Unregister(client.rPeer.id) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed test(5) } diff --git a/les/peer.go b/les/peer.go index eb7452e276..70c863c2ff 100644 --- a/les/peer.go +++ b/les/peer.go @@ -19,7 +19,6 @@ package les import ( "crypto/ecdsa" - "encoding/binary" "errors" "fmt" "math/big" @@ -36,9 +35,10 @@ import ( ) var ( - errClosed = errors.New("peer set is closed") - errAlreadyRegistered = errors.New("peer is already registered") - errNotRegistered = errors.New("peer is not registered") + errClosed = errors.New("peer set is closed") + errAlreadyRegistered = errors.New("peer is already registered") + errNotRegistered = errors.New("peer is not registered") + errInvalidHelpTrieReq = errors.New("invalid help trie request") ) const maxResponseErrors = 50 // number of invalid responses tolerated (makes the protocol less brittle but still avoids spam) @@ -284,21 +284,21 @@ func (p *peer) RequestProofs(reqID, cost uint64, reqs []ProofReq) error { } // RequestHelperTrieProofs fetches a batch of HelperTrie merkle proofs from a remote node. -func (p *peer) RequestHelperTrieProofs(reqID, cost uint64, reqs []HelperTrieReq) error { - p.Log().Debug("Fetching batch of HelperTrie proofs", "count", len(reqs)) +func (p *peer) RequestHelperTrieProofs(reqID, cost uint64, data interface{}) error { switch p.version { case lpv1: - reqsV1 := make([]ChtReq, len(reqs)) - for i, req := range reqs { - if req.Type != htCanonical || req.AuxReq != auxHeader || len(req.Key) != 8 { - return fmt.Errorf("Request invalid in LES/1 mode") - } - blockNum := binary.BigEndian.Uint64(req.Key) - // convert HelperTrie request to old CHT request - reqsV1[i] = ChtReq{ChtNum: (req.TrieIdx + 1) * (light.CHTFrequencyClient / light.CHTFrequencyServer), BlockNum: blockNum, FromLevel: req.FromLevel} + reqs, ok := data.([]ChtReq) + if !ok { + return errInvalidHelpTrieReq } - return sendRequest(p.rw, GetHeaderProofsMsg, reqID, cost, reqsV1) + p.Log().Debug("Fetching batch of header proofs", "count", len(reqs)) + return sendRequest(p.rw, GetHeaderProofsMsg, reqID, cost, reqs) case lpv2: + reqs, ok := data.([]HelperTrieReq) + if !ok { + return errInvalidHelpTrieReq + } + p.Log().Debug("Fetching batch of HelperTrie proofs", "count", len(reqs)) return sendRequest(p.rw, GetHelperTrieProofsMsg, reqID, cost, reqs) default: panic(nil) diff --git a/les/request_test.go b/les/request_test.go index db576798b4..f02c2a3d76 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" ) @@ -84,35 +83,17 @@ func tfCodeAccess(db ethdb.Database, bhash common.Hash, num uint64) light.OdrReq func testAccess(t *testing.T, protocol int, fn accessTestFn) { // Assemble the test environment - peers := newPeerSet() - dist := newRequestDistributor(peers, make(chan struct{})) - rm := newRetrieveManager(peers, dist, nil) - db := ethdb.NewMemDatabase() - ldb := ethdb.NewMemDatabase() - odr := NewLesOdr(ldb, rm) - odr.SetIndexers(light.NewChtIndexer(db, true, nil), light.NewBloomTrieIndexer(db, true, nil), eth.NewBloomIndexer(db, light.BloomTrieFrequency, light.HelperTrieConfirmations)) - - pm := newTestProtocolManagerMust(t, false, 4, testChainGen, nil, nil, db) - lpm := newTestProtocolManagerMust(t, true, 0, nil, peers, odr, ldb) - _, err1, lpeer, err2 := newTestPeerPair("peer", protocol, pm, lpm) - select { - case <-time.After(time.Millisecond * 100): - case err := <-err1: - t.Fatalf("peer 1 handshake error: %v", err) - case err := <-err2: - t.Fatalf("peer 1 handshake error: %v", err) - } - - lpm.synchronise(lpeer) + server, client, tearDown := newClientServerEnv(t, 4, protocol, nil, true) + defer tearDown() + client.pm.synchronise(client.rPeer) test := func(expFail uint64) { - for i := uint64(0); i <= pm.blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := rawdb.ReadCanonicalHash(db, i) - if req := fn(ldb, bhash, i); req != nil { + for i := uint64(0); i <= server.pm.blockchain.CurrentHeader().Number.Uint64(); i++ { + bhash := rawdb.ReadCanonicalHash(server.db, i) + if req := fn(client.db, bhash, i); req != nil { ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() - - err := odr.Retrieve(ctx, req) + err := client.pm.odr.Retrieve(ctx, req) got := err == nil exp := i < expFail if exp && !got { @@ -126,16 +107,16 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { } // temporarily remove peer to test odr fails - peers.Unregister(lpeer.id) + client.peers.Unregister(client.rPeer.id) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed // expect retrievals to fail (except genesis block) without a les peer test(0) - peers.Register(lpeer) + client.peers.Register(client.rPeer) time.Sleep(time.Millisecond * 10) // ensure that all peerSetNotify callbacks are executed - lpeer.lock.Lock() - lpeer.hasBlock = func(common.Hash, uint64) bool { return true } - lpeer.lock.Unlock() + client.rPeer.lock.Lock() + client.rPeer.hasBlock = func(common.Hash, uint64) bool { return true } + client.rPeer.lock.Unlock() // expect all retrievals to pass test(5) } diff --git a/les/server.go b/les/server.go index df98d1e3a8..2fa0456d69 100644 --- a/les/server.go +++ b/les/server.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discv5" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -50,7 +51,7 @@ type LesServer struct { func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { quitSync := make(chan struct{}) - pm, err := NewProtocolManager(eth.BlockChain().Config(), false, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, nil, quitSync, new(sync.WaitGroup)) + pm, err := NewProtocolManager(eth.BlockChain().Config(), light.DefaultServerIndexerConfig, false, config.NetworkId, eth.EventMux(), eth.Engine(), newPeerSet(), eth.BlockChain(), eth.TxPool(), eth.ChainDb(), nil, nil, nil, quitSync, new(sync.WaitGroup)) if err != nil { return nil, err } @@ -64,8 +65,9 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { lesCommons: lesCommons{ config: config, chainDb: eth.ChainDb(), - chtIndexer: light.NewChtIndexer(eth.ChainDb(), false, nil), - bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), false, nil), + iConfig: light.DefaultServerIndexerConfig, + chtIndexer: light.NewChtIndexer(eth.ChainDb(), nil, params.CHTFrequencyServer, params.HelperTrieProcessConfirmations), + bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), nil, params.BloomBitsBlocks, params.BloomTrieFrequency), protocolManager: pm, }, quitSync: quitSync, @@ -75,14 +77,14 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { logger := log.New() chtV1SectionCount, _, _ := srv.chtIndexer.Sections() // indexer still uses LES/1 4k section size for backwards server compatibility - chtV2SectionCount := chtV1SectionCount / (light.CHTFrequencyClient / light.CHTFrequencyServer) + chtV2SectionCount := chtV1SectionCount / (params.CHTFrequencyClient / params.CHTFrequencyServer) if chtV2SectionCount != 0 { // convert to LES/2 section chtLastSection := chtV2SectionCount - 1 // convert last LES/2 section index back to LES/1 index for chtIndexer.SectionHead - chtLastSectionV1 := (chtLastSection+1)*(light.CHTFrequencyClient/light.CHTFrequencyServer) - 1 + chtLastSectionV1 := (chtLastSection+1)*(params.CHTFrequencyClient/params.CHTFrequencyServer) - 1 chtSectionHead := srv.chtIndexer.SectionHead(chtLastSectionV1) - chtRoot := light.GetChtV2Root(pm.chainDb, chtLastSection, chtSectionHead) + chtRoot := light.GetChtRoot(pm.chainDb, chtLastSectionV1, chtSectionHead) logger.Info("Loaded CHT", "section", chtLastSection, "head", chtSectionHead, "root", chtRoot) } bloomTrieSectionCount, _, _ := srv.bloomTrieIndexer.Sections() diff --git a/light/lightchain.go b/light/lightchain.go index b5afe1f0e6..bd798eca24 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -48,6 +48,7 @@ var ( // interface. It only does header validation during chain insertion. type LightChain struct { hc *core.HeaderChain + indexerConfig *IndexerConfig chainDb ethdb.Database odr OdrBackend chainFeed event.Feed @@ -81,13 +82,14 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. blockCache, _ := lru.New(blockCacheLimit) bc := &LightChain{ - chainDb: odr.Database(), - odr: odr, - quit: make(chan struct{}), - bodyCache: bodyCache, - bodyRLPCache: bodyRLPCache, - blockCache: blockCache, - engine: engine, + chainDb: odr.Database(), + indexerConfig: odr.IndexerConfig(), + odr: odr, + quit: make(chan struct{}), + bodyCache: bodyCache, + bodyRLPCache: bodyRLPCache, + blockCache: blockCache, + engine: engine, } var err error bc.hc, err = core.NewHeaderChain(odr.Database(), config, bc.engine, bc.getProcInterrupt) @@ -128,7 +130,7 @@ func (self *LightChain) addTrustedCheckpoint(cp TrustedCheckpoint) { if self.odr.BloomIndexer() != nil { self.odr.BloomIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) } - log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.SectionIdx+1)*CHTFrequencyClient-1, "hash", cp.SectionHead) + log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.SectionIdx+1)*self.indexerConfig.ChtSize-1, "hash", cp.SectionHead) } func (self *LightChain) getProcInterrupt() bool { @@ -472,7 +474,7 @@ func (self *LightChain) SyncCht(ctx context.Context) bool { head := self.CurrentHeader().Number.Uint64() sections, _, _ := self.odr.ChtIndexer().Sections() - latest := sections*CHTFrequencyClient - 1 + latest := sections*self.indexerConfig.ChtSize - 1 if clique := self.hc.Config().Clique; clique != nil { latest -= latest % clique.Epoch // epoch snapshot for clique } diff --git a/light/lightchain_test.go b/light/lightchain_test.go index 5f0baaf4cd..d45c0656d9 100644 --- a/light/lightchain_test.go +++ b/light/lightchain_test.go @@ -55,7 +55,7 @@ func newCanonical(n int) (ethdb.Database, *LightChain, error) { db := ethdb.NewMemDatabase() gspec := core.Genesis{Config: params.TestChainConfig} genesis := gspec.MustCommit(db) - blockchain, _ := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFaker()) + blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker()) // Create and inject the requested chain if n == 0 { @@ -265,7 +265,8 @@ func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types. type dummyOdr struct { OdrBackend - db ethdb.Database + db ethdb.Database + indexerConfig *IndexerConfig } func (odr *dummyOdr) Database() ethdb.Database { @@ -276,6 +277,10 @@ func (odr *dummyOdr) Retrieve(ctx context.Context, req OdrRequest) error { return nil } +func (odr *dummyOdr) IndexerConfig() *IndexerConfig { + return odr.indexerConfig +} + // Tests that reorganizing a long difficult chain after a short easy one // overwrites the canonical numbers and links in the database. func TestReorgLongHeaders(t *testing.T) { diff --git a/light/odr.go b/light/odr.go index 83c64055a2..3cd8b2c040 100644 --- a/light/odr.go +++ b/light/odr.go @@ -44,6 +44,7 @@ type OdrBackend interface { BloomTrieIndexer() *core.ChainIndexer BloomIndexer() *core.ChainIndexer Retrieve(ctx context.Context, req OdrRequest) error + IndexerConfig() *IndexerConfig } // OdrRequest is an interface for retrieval requests @@ -136,6 +137,7 @@ func (req *ReceiptsRequest) StoreResult(db ethdb.Database) { // ChtRequest is the ODR request type for state/storage trie entries type ChtRequest struct { OdrRequest + Config *IndexerConfig ChtNum, BlockNum uint64 ChtRoot common.Hash Header *types.Header @@ -155,6 +157,7 @@ func (req *ChtRequest) StoreResult(db ethdb.Database) { // BloomRequest is the ODR request type for retrieving bloom filters from a CHT structure type BloomRequest struct { OdrRequest + Config *IndexerConfig BloomTrieNum uint64 BitIdx uint SectionIdxList []uint64 @@ -166,7 +169,7 @@ type BloomRequest struct { // StoreResult stores the retrieved data in local database func (req *BloomRequest) StoreResult(db ethdb.Database) { for i, sectionIdx := range req.SectionIdxList { - sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*req.Config.BloomTrieSize-1) // if we don't have the canonical hash stored for this section head number, we'll still store it under // a key with a zero sectionHead. GetBloomBits will look there too if we still don't have the canonical // hash. In the unlikely case we've retrieved the section head hash since then, we'll just retrieve the diff --git a/light/odr_test.go b/light/odr_test.go index 3e7ac10118..eea5b1eab1 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -55,8 +55,9 @@ var ( type testOdr struct { OdrBackend - sdb, ldb ethdb.Database - disable bool + indexerConfig *IndexerConfig + sdb, ldb ethdb.Database + disable bool } func (odr *testOdr) Database() ethdb.Database { @@ -92,6 +93,10 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { return nil } +func (odr *testOdr) IndexerConfig() *IndexerConfig { + return odr.indexerConfig +} + type odrTestFn func(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) func TestOdrGetBlockLes1(t *testing.T) { testChainOdr(t, 1, odrGetBlock) } @@ -258,7 +263,7 @@ func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { t.Fatal(err) } - odr := &testOdr{sdb: sdb, ldb: ldb} + odr := &testOdr{sdb: sdb, ldb: ldb, indexerConfig: TestClientIndexerConfig} lightchain, err := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker()) if err != nil { t.Fatal(err) diff --git a/light/odr_util.go b/light/odr_util.go index 620af63835..9bc0f604b0 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -53,16 +53,16 @@ func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*typ for chtCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) { chtCount-- if chtCount > 0 { - sectionHeadNum = chtCount*CHTFrequencyClient - 1 + sectionHeadNum = chtCount*odr.IndexerConfig().ChtSize - 1 sectionHead = odr.ChtIndexer().SectionHead(chtCount - 1) canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum) } } } - if number >= chtCount*CHTFrequencyClient { + if number >= chtCount*odr.IndexerConfig().ChtSize { return nil, ErrNoTrustedCht } - r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number} + r := &ChtRequest{ChtRoot: GetChtRoot(db, chtCount-1, sectionHead), ChtNum: chtCount - 1, BlockNum: number, Config: odr.IndexerConfig()} if err := odr.Retrieve(ctx, r); err != nil { return nil, err } @@ -175,9 +175,9 @@ func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number // GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to the given bit index and section indexes func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxList []uint64) ([][]byte, error) { - db := odr.Database() - result := make([][]byte, len(sectionIdxList)) var ( + db = odr.Database() + result = make([][]byte, len(sectionIdxList)) reqList []uint64 reqIdx []int ) @@ -193,7 +193,7 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi for bloomTrieCount > 0 && canonicalHash != sectionHead && canonicalHash != (common.Hash{}) { bloomTrieCount-- if bloomTrieCount > 0 { - sectionHeadNum = bloomTrieCount*BloomTrieFrequency - 1 + sectionHeadNum = bloomTrieCount*odr.IndexerConfig().BloomTrieSize - 1 sectionHead = odr.BloomTrieIndexer().SectionHead(bloomTrieCount - 1) canonicalHash = rawdb.ReadCanonicalHash(db, sectionHeadNum) } @@ -201,7 +201,7 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi } for i, sectionIdx := range sectionIdxList { - sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*BloomTrieFrequency-1) + sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*odr.IndexerConfig().BloomSize-1) // if we don't have the canonical hash stored for this section head number, we'll still look for // an entry with a zero sectionHead (we store it with zero section head too if we don't know it // at the time of the retrieval) @@ -209,6 +209,7 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi if err == nil { result[i] = bloomBits } else { + // TODO(rjl493456442) Convert sectionIndex to BloomTrie relative index if sectionIdx >= bloomTrieCount { return nil, ErrNoTrustedBloomTrie } @@ -220,7 +221,8 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bitIdx uint, sectionIdxLi return result, nil } - r := &BloomRequest{BloomTrieRoot: GetBloomTrieRoot(db, bloomTrieCount-1, sectionHead), BloomTrieNum: bloomTrieCount - 1, BitIdx: bitIdx, SectionIdxList: reqList} + r := &BloomRequest{BloomTrieRoot: GetBloomTrieRoot(db, bloomTrieCount-1, sectionHead), BloomTrieNum: bloomTrieCount - 1, + BitIdx: bitIdx, SectionIdxList: reqList, Config: odr.IndexerConfig()} if err := odr.Retrieve(ctx, r); err != nil { return nil, err } else { diff --git a/light/postprocess.go b/light/postprocess.go index f105d57b55..7b23e48b5b 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -36,20 +36,75 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -const ( - // CHTFrequencyClient is the block frequency for creating CHTs on the client side. - CHTFrequencyClient = 32768 +// IndexerConfig includes a set of configs for chain indexers. +type IndexerConfig struct { + // The block frequency for creating CHTs. + ChtSize uint64 - // CHTFrequencyServer is the block frequency for creating CHTs on the server side. - // Eventually this can be merged back with the client version, but that requires a - // full database upgrade, so that should be left for a suitable moment. - CHTFrequencyServer = 4096 + // A special auxiliary field represents client's chtsize for server config, otherwise represents server's chtsize. + PairChtSize uint64 - HelperTrieConfirmations = 2048 // number of confirmations before a server is expected to have the given HelperTrie available - HelperTrieProcessConfirmations = 256 // number of confirmations before a HelperTrie is generated + // The number of confirmations needed to generate/accept a canonical hash help trie. + ChtConfirms uint64 + + // The block frequency for creating new bloom bits. + BloomSize uint64 + + // The number of confirmation needed before a bloom section is considered probably final and its rotated bits + // are calculated. + BloomConfirms uint64 + + // The block frequency for creating BloomTrie. + BloomTrieSize uint64 + + // The number of confirmations needed to generate/accept a bloom trie. + BloomTrieConfirms uint64 +} + +var ( + // DefaultServerIndexerConfig wraps a set of configs as a default indexer config for server side. + DefaultServerIndexerConfig = &IndexerConfig{ + ChtSize: params.CHTFrequencyServer, + PairChtSize: params.CHTFrequencyClient, + ChtConfirms: params.HelperTrieProcessConfirmations, + BloomSize: params.BloomBitsBlocks, + BloomConfirms: params.BloomConfirms, + BloomTrieSize: params.BloomTrieFrequency, + BloomTrieConfirms: params.HelperTrieProcessConfirmations, + } + // DefaultClientIndexerConfig wraps a set of configs as a default indexer config for client side. + DefaultClientIndexerConfig = &IndexerConfig{ + ChtSize: params.CHTFrequencyClient, + PairChtSize: params.CHTFrequencyServer, + ChtConfirms: params.HelperTrieConfirmations, + BloomSize: params.BloomBitsBlocksClient, + BloomConfirms: params.HelperTrieConfirmations, + BloomTrieSize: params.BloomTrieFrequency, + BloomTrieConfirms: params.HelperTrieConfirmations, + } + // TestServerIndexerConfig wraps a set of configs as a test indexer config for server side. + TestServerIndexerConfig = &IndexerConfig{ + ChtSize: 256, + PairChtSize: 2048, + ChtConfirms: 16, + BloomSize: 256, + BloomConfirms: 16, + BloomTrieSize: 2048, + BloomTrieConfirms: 16, + } + // TestClientIndexerConfig wraps a set of configs as a test indexer config for client side. + TestClientIndexerConfig = &IndexerConfig{ + ChtSize: 2048, + PairChtSize: 256, + ChtConfirms: 128, + BloomSize: 2048, + BloomConfirms: 128, + BloomTrieSize: 2048, + BloomTrieConfirms: 128, + } ) -// TrustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with +// trustedCheckpoint represents a set of post-processed trie roots (CHT and BloomTrie) associated with // the appropriate section index and head hash. It is used to start light syncing from this checkpoint // and avoid downloading the entire header chain while still being able to securely access old headers/logs. type TrustedCheckpoint struct { @@ -84,9 +139,9 @@ var trustedCheckpoints = map[common.Hash]TrustedCheckpoint{ } var ( - ErrNoTrustedCht = errors.New("No trusted canonical hash trie") - ErrNoTrustedBloomTrie = errors.New("No trusted bloom trie") - ErrNoHeader = errors.New("Header not found") + ErrNoTrustedCht = errors.New("no trusted canonical hash trie") + ErrNoTrustedBloomTrie = errors.New("no trusted bloom trie") + ErrNoHeader = errors.New("header not found") chtPrefix = []byte("chtRoot-") // chtPrefix + chtNum (uint64 big endian) -> trie root hash ChtTablePrefix = "cht-" ) @@ -97,8 +152,8 @@ type ChtNode struct { Td *big.Int } -// GetChtRoot reads the CHT root assoctiated to the given section from the database -// Note that sectionIdx is specified according to LES/1 CHT section size +// GetChtRoot reads the CHT root associated to the given section from the database +// Note that sectionIdx is specified according to LES/1 CHT section size. func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) @@ -106,21 +161,15 @@ func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) c return common.BytesToHash(data) } -// GetChtV2Root reads the CHT root assoctiated to the given section from the database -// Note that sectionIdx is specified according to LES/2 CHT section size -func GetChtV2Root(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { - return GetChtRoot(db, (sectionIdx+1)*(CHTFrequencyClient/CHTFrequencyServer)-1, sectionHead) -} - -// StoreChtRoot writes the CHT root assoctiated to the given section into the database -// Note that sectionIdx is specified according to LES/1 CHT section size +// StoreChtRoot writes the CHT root associated to the given section into the database +// Note that sectionIdx is specified according to LES/1 CHT section size. func StoreChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) db.Put(append(append(chtPrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) } -// ChtIndexerBackend implements core.ChainIndexerBackend +// ChtIndexerBackend implements core.ChainIndexerBackend. type ChtIndexerBackend struct { diskdb, trieTable ethdb.Database odr OdrBackend @@ -130,33 +179,24 @@ type ChtIndexerBackend struct { trie *trie.Trie } -// NewBloomTrieIndexer creates a BloomTrie chain indexer -func NewChtIndexer(db ethdb.Database, clientMode bool, odr OdrBackend) *core.ChainIndexer { - var sectionSize, confirmReq uint64 - if clientMode { - sectionSize = CHTFrequencyClient - confirmReq = HelperTrieConfirmations - } else { - sectionSize = CHTFrequencyServer - confirmReq = HelperTrieProcessConfirmations - } - idb := ethdb.NewTable(db, "chtIndex-") +// NewChtIndexer creates a Cht chain indexer +func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64) *core.ChainIndexer { trieTable := ethdb.NewTable(db, ChtTablePrefix) backend := &ChtIndexerBackend{ diskdb: db, odr: odr, trieTable: trieTable, triedb: trie.NewDatabase(trieTable), - sectionSize: sectionSize, + sectionSize: size, } - return core.NewChainIndexer(db, idb, backend, sectionSize, confirmReq, time.Millisecond*100, "cht") + return core.NewChainIndexer(db, ethdb.NewTable(db, "chtIndex-"), backend, size, confirms, time.Millisecond*100, "cht") } // fetchMissingNodes tries to retrieve the last entry of the latest trusted CHT from the // ODR backend in order to be able to add new entries and calculate subsequent root hashes func (c *ChtIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error { batch := c.trieTable.NewBatch() - r := &ChtRequest{ChtRoot: root, ChtNum: section - 1, BlockNum: section*c.sectionSize - 1} + r := &ChtRequest{ChtRoot: root, ChtNum: section - 1, BlockNum: section*c.sectionSize - 1, Config: c.odr.IndexerConfig()} for { err := c.odr.Retrieve(ctx, r) switch err { @@ -221,18 +261,13 @@ func (c *ChtIndexerBackend) Commit() error { } c.triedb.Commit(root, false) - if ((c.section+1)*c.sectionSize)%CHTFrequencyClient == 0 { - log.Info("Storing CHT", "section", c.section*c.sectionSize/CHTFrequencyClient, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) + if ((c.section+1)*c.sectionSize)%params.CHTFrequencyClient == 0 { + log.Info("Storing CHT", "section", c.section*c.sectionSize/params.CHTFrequencyClient, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) } StoreChtRoot(c.diskdb, c.section, c.lastHash, root) return nil } -const ( - BloomTrieFrequency = 32768 - ethBloomBitsSection = 4096 -) - var ( bloomTriePrefix = []byte("bltRoot-") // bloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash BloomTrieTablePrefix = "blt-" @@ -255,33 +290,31 @@ func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root // BloomTrieIndexerBackend implements core.ChainIndexerBackend type BloomTrieIndexerBackend struct { - diskdb, trieTable ethdb.Database - odr OdrBackend - triedb *trie.Database - section, parentSectionSize, bloomTrieRatio uint64 - trie *trie.Trie - sectionHeads []common.Hash + diskdb, trieTable ethdb.Database + triedb *trie.Database + odr OdrBackend + section uint64 + parentSize uint64 + size uint64 + bloomTrieRatio uint64 + trie *trie.Trie + sectionHeads []common.Hash } // NewBloomTrieIndexer creates a BloomTrie chain indexer -func NewBloomTrieIndexer(db ethdb.Database, clientMode bool, odr OdrBackend) *core.ChainIndexer { +func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uint64) *core.ChainIndexer { trieTable := ethdb.NewTable(db, BloomTrieTablePrefix) backend := &BloomTrieIndexerBackend{ - diskdb: db, - odr: odr, - trieTable: trieTable, - triedb: trie.NewDatabase(trieTable), + diskdb: db, + odr: odr, + trieTable: trieTable, + triedb: trie.NewDatabase(trieTable), + parentSize: parentSize, + size: size, } - idb := ethdb.NewTable(db, "bltIndex-") - - if clientMode { - backend.parentSectionSize = BloomTrieFrequency - } else { - backend.parentSectionSize = ethBloomBitsSection - } - backend.bloomTrieRatio = BloomTrieFrequency / backend.parentSectionSize + backend.bloomTrieRatio = size / parentSize backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) - return core.NewChainIndexer(db, idb, backend, BloomTrieFrequency, 0, time.Millisecond*100, "bloomtrie") + return core.NewChainIndexer(db, ethdb.NewTable(db, "bltIndex-"), backend, size, 0, time.Millisecond*100, "bloomtrie") } // fetchMissingNodes tries to retrieve the last entries of the latest trusted bloom trie from the @@ -296,7 +329,7 @@ func (b *BloomTrieIndexerBackend) fetchMissingNodes(ctx context.Context, section for i := 0; i < 20; i++ { go func() { for bitIndex := range indexCh { - r := &BloomRequest{BloomTrieRoot: root, BloomTrieNum: section - 1, BitIdx: bitIndex, SectionIdxList: []uint64{section - 1}} + r := &BloomRequest{BloomTrieRoot: root, BloomTrieNum: section - 1, BitIdx: bitIndex, SectionIdxList: []uint64{section - 1}, Config: b.odr.IndexerConfig()} for { if err := b.odr.Retrieve(ctx, r); err == ErrNoPeers { // if there are no peers to serve, retry later @@ -351,9 +384,9 @@ func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, las // Process implements core.ChainIndexerBackend func (b *BloomTrieIndexerBackend) Process(ctx context.Context, header *types.Header) error { - num := header.Number.Uint64() - b.section*BloomTrieFrequency - if (num+1)%b.parentSectionSize == 0 { - b.sectionHeads[num/b.parentSectionSize] = header.Hash() + num := header.Number.Uint64() - b.section*b.size + if (num+1)%b.parentSize == 0 { + b.sectionHeads[num/b.parentSize] = header.Hash() } return nil } @@ -372,7 +405,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { if err != nil { return err } - decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSectionSize/8)) + decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSize/8)) if err2 != nil { return err2 } @@ -397,6 +430,5 @@ func (b *BloomTrieIndexerBackend) Commit() error { sectionHead := b.sectionHeads[b.bloomTrieRatio-1] log.Info("Storing bloom trie", "section", b.section, "head", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression", float64(compSize)/float64(decompSize)) StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) - return nil } diff --git a/light/trie_test.go b/light/trie_test.go index 84c6f162fb..6bddfefe28 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -47,7 +47,7 @@ func TestNodeIterator(t *testing.T) { } ctx := context.Background() - odr := &testOdr{sdb: fulldb, ldb: lightdb} + odr := &testOdr{sdb: fulldb, ldb: lightdb, indexerConfig: TestClientIndexerConfig} head := blockchain.CurrentHeader() lightTrie, _ := NewStateDatabase(ctx, head, odr).OpenTrie(head.Root) fullTrie, _ := state.NewDatabase(fulldb).OpenTrie(head.Root) diff --git a/light/txpool_test.go b/light/txpool_test.go index ccbd83a943..204347a6e4 100644 --- a/light/txpool_test.go +++ b/light/txpool_test.go @@ -94,7 +94,7 @@ func TestTxPool(t *testing.T) { panic(err) } - odr := &testOdr{sdb: sdb, ldb: ldb} + odr := &testOdr{sdb: sdb, ldb: ldb, indexerConfig: TestClientIndexerConfig} relay := &testTxRelay{ send: make(chan int, 1), discard: make(chan int, 1), diff --git a/params/network_params.go b/params/network_params.go index 536a46d3df..f8731e8979 100644 --- a/params/network_params.go +++ b/params/network_params.go @@ -17,10 +17,38 @@ package params // These are network parameters that need to be constant between clients, but -// aren't necesarilly consensus related. +// aren't necessarily consensus related. const ( // BloomBitsBlocks is the number of blocks a single bloom bit section vector - // contains. + // contains on the server side. BloomBitsBlocks uint64 = 4096 + + // BloomBitsBlocksClient is the number of blocks a single bloom bit section vector + // contains on the light client side + BloomBitsBlocksClient uint64 = 32768 + + // BloomConfirms is the number of confirmation blocks before a bloom section is + // considered probably final and its rotated bits are calculated. + BloomConfirms = 256 + + // CHTFrequencyClient is the block frequency for creating CHTs on the client side. + CHTFrequencyClient = 32768 + + // CHTFrequencyServer is the block frequency for creating CHTs on the server side. + // Eventually this can be merged back with the client version, but that requires a + // full database upgrade, so that should be left for a suitable moment. + CHTFrequencyServer = 4096 + + // BloomTrieFrequency is the block frequency for creating BloomTrie on both + // server/client sides. + BloomTrieFrequency = 32768 + + // HelperTrieConfirmations is the number of confirmations before a client is expected + // to have the given HelperTrie available. + HelperTrieConfirmations = 2048 + + // HelperTrieProcessConfirmations is the number of confirmations before a HelperTrie + // is generated + HelperTrieProcessConfirmations = 256 ) From 63352bf4247f05d8ef255ff8c63290225c3bc671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 28 Aug 2018 09:31:34 +0200 Subject: [PATCH 62/96] core: safe indexer operation when syncing starts before the checkpoint (#17511) --- core/chain_indexer.go | 56 ++++++++++++++++++++++++++++++++++--------- light/lightchain.go | 6 ++--- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/core/chain_indexer.go b/core/chain_indexer.go index b80b517d91..89ee75eb29 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -85,6 +85,9 @@ type ChainIndexer struct { knownSections uint64 // Number of sections known to be complete (block wise) cascadedHead uint64 // Block number of the last completed section cascaded to subindexers + checkpointSections uint64 // Number of sections covered by the checkpoint + checkpointHead common.Hash // Section head belonging to the checkpoint + throttling time.Duration // Disk throttling to prevent a heavy upgrade from hogging resources log log.Logger @@ -115,12 +118,19 @@ func NewChainIndexer(chainDb, indexDb ethdb.Database, backend ChainIndexerBacken return c } -// AddKnownSectionHead marks a new section head as known/processed if it is newer -// than the already known best section head -func (c *ChainIndexer) AddKnownSectionHead(section uint64, shead common.Hash) { +// AddCheckpoint adds a checkpoint. Sections are never processed and the chain +// is not expected to be available before this point. The indexer assumes that +// the backend has sufficient information available to process subsequent sections. +// +// Note: knownSections == 0 and storedSections == checkpointSections until +// syncing reaches the checkpoint +func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) { c.lock.Lock() defer c.lock.Unlock() + c.checkpointSections = section + 1 + c.checkpointHead = shead + if section < c.storedSections { return } @@ -233,16 +243,23 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) { // If a reorg happened, invalidate all sections until that point if reorg { // Revert the known section number to the reorg point - changed := head / c.sectionSize - if changed < c.knownSections { - c.knownSections = changed + known := head / c.sectionSize + stored := known + if known < c.checkpointSections { + known = 0 + } + if stored < c.checkpointSections { + stored = c.checkpointSections + } + if known < c.knownSections { + c.knownSections = known } // Revert the stored sections from the database to the reorg point - if changed < c.storedSections { - c.setValidSections(changed) + if stored < c.storedSections { + c.setValidSections(stored) } // Update the new head number to the finalized section end and notify children - head = changed * c.sectionSize + head = known * c.sectionSize if head < c.cascadedHead { c.cascadedHead = head @@ -256,7 +273,18 @@ func (c *ChainIndexer) newHead(head uint64, reorg bool) { var sections uint64 if head >= c.confirmsReq { sections = (head + 1 - c.confirmsReq) / c.sectionSize + if sections < c.checkpointSections { + sections = 0 + } if sections > c.knownSections { + if c.knownSections < c.checkpointSections { + // syncing reached the checkpoint, verify section head + syncedHead := rawdb.ReadCanonicalHash(c.chainDb, c.checkpointSections*c.sectionSize-1) + if syncedHead != c.checkpointHead { + c.log.Error("Synced chain does not match checkpoint", "number", c.checkpointSections*c.sectionSize-1, "expected", c.checkpointHead, "synced", syncedHead) + return + } + } c.knownSections = sections select { @@ -401,8 +429,14 @@ func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) { c.children = append(c.children, indexer) // Cascade any pending updates to new children too - if c.storedSections > 0 { - indexer.newHead(c.storedSections*c.sectionSize-1, false) + sections := c.storedSections + if c.knownSections < sections { + // if a section is "stored" but not "known" then it is a checkpoint without + // available chain data so we should not cascade it yet + sections = c.knownSections + } + if sections > 0 { + indexer.newHead(sections*c.sectionSize-1, false) } } diff --git a/light/lightchain.go b/light/lightchain.go index bd798eca24..d40a4ee6c6 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -121,14 +121,14 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. func (self *LightChain) addTrustedCheckpoint(cp TrustedCheckpoint) { if self.odr.ChtIndexer() != nil { StoreChtRoot(self.chainDb, cp.SectionIdx, cp.SectionHead, cp.CHTRoot) - self.odr.ChtIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) + self.odr.ChtIndexer().AddCheckpoint(cp.SectionIdx, cp.SectionHead) } if self.odr.BloomTrieIndexer() != nil { StoreBloomTrieRoot(self.chainDb, cp.SectionIdx, cp.SectionHead, cp.BloomRoot) - self.odr.BloomTrieIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) + self.odr.BloomTrieIndexer().AddCheckpoint(cp.SectionIdx, cp.SectionHead) } if self.odr.BloomIndexer() != nil { - self.odr.BloomIndexer().AddKnownSectionHead(cp.SectionIdx, cp.SectionHead) + self.odr.BloomIndexer().AddCheckpoint(cp.SectionIdx, cp.SectionHead) } log.Info("Added trusted checkpoint", "chain", cp.name, "block", (cp.SectionIdx+1)*self.indexerConfig.ChtSize-1, "hash", cp.SectionHead) } From c1c003e4ff36c22d67662ca661fc78cde850d401 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 28 Aug 2018 21:59:05 +0800 Subject: [PATCH 63/96] consensus, miner: stale block mining support (#17506) * consensus, miner: stale block supporting * consensus, miner: refactor seal signature * cmd, consensus, eth: add miner noverify flag * cmd, consensus, miner: polish --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 11 ++- consensus/clique/clique.go | 39 +++++---- consensus/consensus.go | 9 +- consensus/ethash/algorithm_test.go | 2 +- consensus/ethash/ethash.go | 33 +++---- consensus/ethash/ethash_test.go | 40 ++++----- consensus/ethash/sealer.go | 134 +++++++++++++++++++---------- consensus/ethash/sealer_test.go | 94 ++++++++++++++++++-- eth/api_tracer.go | 2 +- eth/backend.go | 8 +- eth/config.go | 1 + eth/gen_config.go | 7 +- les/backend.go | 2 +- miner/worker.go | 116 +++++++++++-------------- 16 files changed, 317 insertions(+), 183 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 7b8a8244ee..0ed67d0d57 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -108,6 +108,7 @@ var ( utils.MinerExtraDataFlag, utils.MinerLegacyExtraDataFlag, utils.MinerRecommitIntervalFlag, + utils.MinerNoVerfiyFlag, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 1e27d0ae8d..2f8260e0f6 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -192,6 +192,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, + utils.MinerNoVerfiyFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b9a33ffe70..13430ad565 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -366,9 +366,13 @@ var ( } MinerRecommitIntervalFlag = cli.DurationFlag{ Name: "miner.recommit", - Usage: "Time interval to recreate the block being mined.", + Usage: "Time interval to recreate the block being mined", Value: eth.DefaultConfig.MinerRecommit, } + MinerNoVerfiyFlag = cli.BoolFlag{ + Name: "miner.noverify", + Usage: "Disable remote sealing verification", + } // Account settings UnlockedAccountFlag = cli.StringFlag{ Name: "unlock", @@ -1151,6 +1155,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(MinerRecommitIntervalFlag.Name) { cfg.MinerRecommit = ctx.Duration(MinerRecommitIntervalFlag.Name) } + if ctx.GlobalIsSet(MinerNoVerfiyFlag.Name) { + cfg.MinerNoverify = ctx.Bool(MinerNoVerfiyFlag.Name) + } if ctx.GlobalIsSet(VMEnableDebugFlag.Name) { // TODO(fjl): force-enable this in --dev mode cfg.EnablePreimageRecording = ctx.GlobalBool(VMEnableDebugFlag.Name) @@ -1345,7 +1352,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai DatasetDir: stack.ResolvePath(eth.DefaultConfig.Ethash.DatasetDir), DatasetsInMem: eth.DefaultConfig.Ethash.DatasetsInMem, DatasetsOnDisk: eth.DefaultConfig.Ethash.DatasetsOnDisk, - }, nil) + }, nil, false) } } if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 3730c91f62..5472909846 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -590,17 +590,17 @@ func (c *Clique) Authorize(signer common.Address, signFn SignerFn) { // Seal implements consensus.Engine, attempting to create a sealed block using // the local signing credentials. -func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) { +func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error { header := block.Header() // Sealing the genesis block is not supported number := header.Number.Uint64() if number == 0 { - return nil, errUnknownBlock + return errUnknownBlock } // For 0-period chains, refuse to seal empty blocks (no reward but would spin sealing) if c.config.Period == 0 && len(block.Transactions()) == 0 { - return nil, errWaitTransactions + return errWaitTransactions } // Don't hold the signer fields for the entire sealing procedure c.lock.RLock() @@ -610,10 +610,10 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch // Bail out if we're unauthorized to sign a block snap, err := c.snapshot(chain, number-1, header.ParentHash, nil) if err != nil { - return nil, err + return err } if _, authorized := snap.Signers[signer]; !authorized { - return nil, errUnauthorized + return errUnauthorized } // If we're amongst the recent signers, wait for the next block for seen, recent := range snap.Recents { @@ -621,8 +621,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch // Signer is among recents, only wait if the current block doesn't shift it out if limit := uint64(len(snap.Signers)/2 + 1); number < limit || seen > number-limit { log.Info("Signed recently, must wait for others") - <-stop - return nil, nil + return nil } } } @@ -635,21 +634,29 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch log.Trace("Out-of-turn signing requested", "wiggle", common.PrettyDuration(wiggle)) } - log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay)) - - select { - case <-stop: - return nil, nil - case <-time.After(delay): - } // Sign all the things! sighash, err := signFn(accounts.Account{Address: signer}, sigHash(header).Bytes()) if err != nil { - return nil, err + return err } copy(header.Extra[len(header.Extra)-extraSeal:], sighash) + // Wait until sealing is terminated or delay timeout. + log.Trace("Waiting for slot to sign and propagate", "delay", common.PrettyDuration(delay)) + go func() { + select { + case <-stop: + return + case <-time.After(delay): + } - return block.WithSeal(header), nil + select { + case results <- block.WithSeal(header): + default: + log.Warn("Sealing result is not read by miner", "sealhash", c.SealHash(header)) + } + }() + + return nil } // CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty diff --git a/consensus/consensus.go b/consensus/consensus.go index 27799f13c0..12ede7ff46 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -86,9 +86,12 @@ type Engine interface { Finalize(chain ChainReader, header *types.Header, state *state.StateDB, txs []*types.Transaction, uncles []*types.Header, receipts []*types.Receipt) (*types.Block, error) - // Seal generates a new block for the given input block with the local miner's - // seal place on top. - Seal(chain ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) + // Seal generates a new sealing request for the given input block and pushes + // the result into the given channel. + // + // Note, the method returns immediately and will send the result async. More + // than one result may also be returned depending on the consensus algorothm. + Seal(chain ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error // SealHash returns the hash of a block prior to it being sealed. SealHash(header *types.Header) common.Hash diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go index db22cccd0c..c58479e289 100644 --- a/consensus/ethash/algorithm_test.go +++ b/consensus/ethash/algorithm_test.go @@ -729,7 +729,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) { go func(idx int) { defer pend.Done() - ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}, nil) + ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}, nil, false) defer ethash.Close() if err := ethash.VerifySeal(nil, block.Header()); err != nil { t.Errorf("proc %d: block verification failed: %v", idx, err) diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index d98c3371c5..b4819ca38b 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -50,7 +50,7 @@ var ( two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) // sharedEthash is a full instance that can be shared between multiple users. - sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}, nil) + sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}, nil, false) // algorithmRevision is the data structure version used for file naming. algorithmRevision = 23 @@ -405,6 +405,12 @@ type Config struct { PowMode Mode } +// sealTask wraps a seal block with relative result channel for remote sealer thread. +type sealTask struct { + block *types.Block + results chan<- *types.Block +} + // mineResult wraps the pow solution parameters for the specified block. type mineResult struct { nonce types.BlockNonce @@ -444,12 +450,11 @@ type Ethash struct { hashrate metrics.Meter // Meter tracking the average hashrate // Remote sealer related fields - workCh chan *types.Block // Notification channel to push new work to remote sealer - resultCh chan *types.Block // Channel used by mining threads to return result - fetchWorkCh chan *sealWork // Channel used for remote sealer to fetch mining work - submitWorkCh chan *mineResult // Channel used for remote sealer to submit their mining result - fetchRateCh chan chan uint64 // Channel used to gather submitted hash rate for local or remote sealer. - submitRateCh chan *hashrate // Channel used for remote sealer to submit their mining hashrate + workCh chan *sealTask // Notification channel to push new work and relative result channel to remote sealer + fetchWorkCh chan *sealWork // Channel used for remote sealer to fetch mining work + submitWorkCh chan *mineResult // Channel used for remote sealer to submit their mining result + fetchRateCh chan chan uint64 // Channel used to gather submitted hash rate for local or remote sealer. + submitRateCh chan *hashrate // Channel used for remote sealer to submit their mining hashrate // The fields below are hooks for testing shared *Ethash // Shared PoW verifier to avoid cache regeneration @@ -464,7 +469,7 @@ type Ethash struct { // New creates a full sized ethash PoW scheme and starts a background thread for // remote mining, also optionally notifying a batch of remote services of new work // packages. -func New(config Config, notify []string) *Ethash { +func New(config Config, notify []string, noverify bool) *Ethash { if config.CachesInMem <= 0 { log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem) config.CachesInMem = 1 @@ -481,36 +486,34 @@ func New(config Config, notify []string) *Ethash { datasets: newlru("dataset", config.DatasetsInMem, newDataset), update: make(chan struct{}), hashrate: metrics.NewMeter(), - workCh: make(chan *types.Block), - resultCh: make(chan *types.Block), + workCh: make(chan *sealTask), fetchWorkCh: make(chan *sealWork), submitWorkCh: make(chan *mineResult), fetchRateCh: make(chan chan uint64), submitRateCh: make(chan *hashrate), exitCh: make(chan chan error), } - go ethash.remote(notify) + go ethash.remote(notify, noverify) return ethash } // NewTester creates a small sized ethash PoW scheme useful only for testing // purposes. -func NewTester(notify []string) *Ethash { +func NewTester(notify []string, noverify bool) *Ethash { ethash := &Ethash{ config: Config{PowMode: ModeTest}, caches: newlru("cache", 1, newCache), datasets: newlru("dataset", 1, newDataset), update: make(chan struct{}), hashrate: metrics.NewMeter(), - workCh: make(chan *types.Block), - resultCh: make(chan *types.Block), + workCh: make(chan *sealTask), fetchWorkCh: make(chan *sealWork), submitWorkCh: make(chan *mineResult), fetchRateCh: make(chan chan uint64), submitRateCh: make(chan *hashrate), exitCh: make(chan chan error), } - go ethash.remote(notify) + go ethash.remote(notify, noverify) return ethash } diff --git a/consensus/ethash/ethash_test.go b/consensus/ethash/ethash_test.go index b190d63d6b..8eded2ca81 100644 --- a/consensus/ethash/ethash_test.go +++ b/consensus/ethash/ethash_test.go @@ -34,17 +34,23 @@ import ( func TestTestMode(t *testing.T) { header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} - ethash := NewTester(nil) + ethash := NewTester(nil, false) defer ethash.Close() - block, err := ethash.Seal(nil, types.NewBlockWithHeader(header), nil) + results := make(chan *types.Block) + err := ethash.Seal(nil, types.NewBlockWithHeader(header), results, nil) if err != nil { t.Fatalf("failed to seal block: %v", err) } - header.Nonce = types.EncodeNonce(block.Nonce()) - header.MixDigest = block.MixDigest() - if err := ethash.VerifySeal(nil, header); err != nil { - t.Fatalf("unexpected verification error: %v", err) + select { + case block := <-results: + header.Nonce = types.EncodeNonce(block.Nonce()) + header.MixDigest = block.MixDigest() + if err := ethash.VerifySeal(nil, header); err != nil { + t.Fatalf("unexpected verification error: %v", err) + } + case <-time.NewTimer(time.Second).C: + t.Error("sealing result timeout") } } @@ -56,7 +62,7 @@ func TestCacheFileEvict(t *testing.T) { t.Fatal(err) } defer os.RemoveAll(tmpdir) - e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil) + e := New(Config{CachesInMem: 3, CachesOnDisk: 10, CacheDir: tmpdir, PowMode: ModeTest}, nil, false) defer e.Close() workers := 8 @@ -85,7 +91,7 @@ func verifyTest(wg *sync.WaitGroup, e *Ethash, workerIndex, epochs int) { } func TestRemoteSealer(t *testing.T) { - ethash := NewTester(nil) + ethash := NewTester(nil, false) defer ethash.Close() api := &API{ethash} @@ -97,7 +103,8 @@ func TestRemoteSealer(t *testing.T) { sealhash := ethash.SealHash(header) // Push new work. - ethash.Seal(nil, block, nil) + results := make(chan *types.Block) + ethash.Seal(nil, block, results, nil) var ( work [3]string @@ -114,20 +121,11 @@ func TestRemoteSealer(t *testing.T) { header = &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(1000)} block = types.NewBlockWithHeader(header) sealhash = ethash.SealHash(header) - ethash.Seal(nil, block, nil) + ethash.Seal(nil, block, results, nil) if work, err = api.GetWork(); err != nil || work[0] != sealhash.Hex() { t.Error("expect to return the latest pushed work") } - // Push block with higher block number. - newHead := &types.Header{Number: big.NewInt(2), Difficulty: big.NewInt(100)} - newBlock := types.NewBlockWithHeader(newHead) - newSealhash := ethash.SealHash(newHead) - ethash.Seal(nil, newBlock, nil) - - if res := api.SubmitWork(types.BlockNonce{}, newSealhash, common.Hash{}); res { - t.Error("expect to return false when submit a stale solution") - } } func TestHashRate(t *testing.T) { @@ -136,7 +134,7 @@ func TestHashRate(t *testing.T) { expect uint64 ids = []common.Hash{common.HexToHash("a"), common.HexToHash("b"), common.HexToHash("c")} ) - ethash := NewTester(nil) + ethash := NewTester(nil, false) defer ethash.Close() if tot := ethash.Hashrate(); tot != 0 { @@ -156,7 +154,7 @@ func TestHashRate(t *testing.T) { } func TestClosedRemoteSealer(t *testing.T) { - ethash := NewTester(nil) + ethash := NewTester(nil, false) time.Sleep(1 * time.Second) // ensure exit channel is listening ethash.Close() diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index a458c60f66..06c98a7811 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -35,6 +35,11 @@ import ( "github.com/ethereum/go-ethereum/log" ) +const ( + // staleThreshold is the maximum depth of the acceptable stale but valid ethash solution. + staleThreshold = 7 +) + var ( errNoMiningWork = errors.New("no mining work available yet") errInvalidSealResult = errors.New("invalid or stale proof-of-work solution") @@ -42,16 +47,21 @@ var ( // Seal implements consensus.Engine, attempting to find a nonce that satisfies // the block's difficulty requirements. -func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop <-chan struct{}) (*types.Block, error) { +func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error { // If we're running a fake PoW, simply return a 0 nonce immediately if ethash.config.PowMode == ModeFake || ethash.config.PowMode == ModeFullFake { header := block.Header() header.Nonce, header.MixDigest = types.BlockNonce{}, common.Hash{} - return block.WithSeal(header), nil + select { + case results <- block.WithSeal(header): + default: + log.Warn("Sealing result is not read by miner", "mode", "fake", "sealhash", ethash.SealHash(block.Header())) + } + return nil } // If we're running a shared PoW, delegate sealing to it if ethash.shared != nil { - return ethash.shared.Seal(chain, block, stop) + return ethash.shared.Seal(chain, block, results, stop) } // Create a runner and the multiple search threads it directs abort := make(chan struct{}) @@ -62,7 +72,7 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64)) if err != nil { ethash.lock.Unlock() - return nil, err + return err } ethash.rand = rand.New(rand.NewSource(seed.Int64())) } @@ -75,34 +85,45 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, stop } // Push new work to remote sealer if ethash.workCh != nil { - ethash.workCh <- block + ethash.workCh <- &sealTask{block: block, results: results} } - var pend sync.WaitGroup + var ( + pend sync.WaitGroup + locals = make(chan *types.Block) + ) for i := 0; i < threads; i++ { pend.Add(1) go func(id int, nonce uint64) { defer pend.Done() - ethash.mine(block, id, nonce, abort, ethash.resultCh) + ethash.mine(block, id, nonce, abort, locals) }(i, uint64(ethash.rand.Int63())) } // Wait until sealing is terminated or a nonce is found - var result *types.Block - select { - case <-stop: - // Outside abort, stop all miner threads - close(abort) - case result = <-ethash.resultCh: - // One of the threads found a block, abort all others - close(abort) - case <-ethash.update: - // Thread count was changed on user request, restart - close(abort) + go func() { + var result *types.Block + select { + case <-stop: + // Outside abort, stop all miner threads + close(abort) + case result = <-locals: + // One of the threads found a block, abort all others + select { + case results <- result: + default: + log.Warn("Sealing result is not read by miner", "mode", "local", "sealhash", ethash.SealHash(block.Header())) + } + close(abort) + case <-ethash.update: + // Thread count was changed on user request, restart + close(abort) + if err := ethash.Seal(chain, block, results, stop); err != nil { + log.Error("Failed to restart sealing after update", "err", err) + } + } + // Wait for all miners to terminate and return the block pend.Wait() - return ethash.Seal(chain, block, stop) - } - // Wait for all miners to terminate and return the block - pend.Wait() - return result, nil + }() + return nil } // mine is the actual proof-of-work miner that searches for a nonce starting from @@ -165,11 +186,12 @@ search: } // remote is a standalone goroutine to handle remote mining related stuff. -func (ethash *Ethash) remote(notify []string) { +func (ethash *Ethash) remote(notify []string, noverify bool) { var ( works = make(map[common.Hash]*types.Block) rates = make(map[common.Hash]hashrate) + results chan<- *types.Block currentBlock *types.Block currentWork [3]string @@ -226,11 +248,15 @@ func (ethash *Ethash) remote(notify []string) { // submitWork verifies the submitted pow solution, returning // whether the solution was accepted or not (not can be both a bad pow as well as // any other error, like no pending work or stale mining result). - submitWork := func(nonce types.BlockNonce, mixDigest common.Hash, hash common.Hash) bool { + submitWork := func(nonce types.BlockNonce, mixDigest common.Hash, sealhash common.Hash) bool { + if currentBlock == nil { + log.Error("Pending work without block", "sealhash", sealhash) + return false + } // Make sure the work submitted is present - block := works[hash] + block := works[sealhash] if block == nil { - log.Info("Work submitted but none pending", "hash", hash) + log.Warn("Work submitted but none pending", "sealhash", sealhash, "curnumber", currentBlock.NumberU64()) return false } // Verify the correctness of submitted result. @@ -239,26 +265,36 @@ func (ethash *Ethash) remote(notify []string) { header.MixDigest = mixDigest start := time.Now() - if err := ethash.verifySeal(nil, header, true); err != nil { - log.Warn("Invalid proof-of-work submitted", "hash", hash, "elapsed", time.Since(start), "err", err) - return false + if !noverify { + if err := ethash.verifySeal(nil, header, true); err != nil { + log.Warn("Invalid proof-of-work submitted", "sealhash", sealhash, "elapsed", time.Since(start), "err", err) + return false + } } - // Make sure the result channel is created. - if ethash.resultCh == nil { + // Make sure the result channel is assigned. + if results == nil { log.Warn("Ethash result channel is empty, submitted mining result is rejected") return false } - log.Trace("Verified correct proof-of-work", "hash", hash, "elapsed", time.Since(start)) + log.Trace("Verified correct proof-of-work", "sealhash", sealhash, "elapsed", time.Since(start)) // Solutions seems to be valid, return to the miner and notify acceptance. - select { - case ethash.resultCh <- block.WithSeal(header): - delete(works, hash) - return true - default: - log.Info("Work submitted is stale", "hash", hash) - return false + solution := block.WithSeal(header) + + // The submitted solution is within the scope of acceptance. + if solution.NumberU64()+staleThreshold > currentBlock.NumberU64() { + select { + case results <- solution: + log.Debug("Work submitted is acceptable", "number", solution.NumberU64(), "sealhash", sealhash, "hash", solution.Hash()) + return true + default: + log.Warn("Sealing result is not read by miner", "mode", "remote", "sealhash", sealhash) + return false + } } + // The submitted block is too old to accept, drop it. + log.Warn("Work submitted is too old", "number", solution.NumberU64(), "sealhash", sealhash, "hash", solution.Hash()) + return false } ticker := time.NewTicker(5 * time.Second) @@ -266,14 +302,12 @@ func (ethash *Ethash) remote(notify []string) { for { select { - case block := <-ethash.workCh: - if currentBlock != nil && block.ParentHash() != currentBlock.ParentHash() { - // Start new round mining, throw out all previous work. - works = make(map[common.Hash]*types.Block) - } + case work := <-ethash.workCh: // Update current work with new received block. // Note same work can be past twice, happens when changing CPU threads. - makeWork(block) + results = work.results + + makeWork(work.block) // Notify and requested URLs of the new work availability notifyWork() @@ -315,6 +349,14 @@ func (ethash *Ethash) remote(notify []string) { delete(rates, id) } } + // Clear stale pending blocks + if currentBlock != nil { + for hash, block := range works { + if block.NumberU64()+staleThreshold <= currentBlock.NumberU64() { + delete(works, hash) + } + } + } case errc := <-ethash.exitCh: // Exit remote loop if ethash is closed and return relevant error. diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go index d1b66f9cf3..31d18b67c7 100644 --- a/consensus/ethash/sealer_test.go +++ b/consensus/ethash/sealer_test.go @@ -41,14 +41,14 @@ func TestRemoteNotify(t *testing.T) { go server.Serve(listener) // Create the custom ethash engine - ethash := NewTester([]string{"http://" + listener.Addr().String()}) + ethash := NewTester([]string{"http://" + listener.Addr().String()}, false) defer ethash.Close() // Stream a work task and ensure the notification bubbles out header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} block := types.NewBlockWithHeader(header) - ethash.Seal(nil, block, nil) + ethash.Seal(nil, block, nil, nil) select { case work := <-sink: if want := ethash.SealHash(header).Hex(); work[0] != want { @@ -66,7 +66,7 @@ func TestRemoteNotify(t *testing.T) { } } -// Tests that pushing work packages fast to the miner doesn't cause any daa race +// Tests that pushing work packages fast to the miner doesn't cause any data race // issues in the notifications. func TestRemoteMultiNotify(t *testing.T) { // Start a simple webserver to capture notifications @@ -95,7 +95,7 @@ func TestRemoteMultiNotify(t *testing.T) { go server.Serve(listener) // Create the custom ethash engine - ethash := NewTester([]string{"http://" + listener.Addr().String()}) + ethash := NewTester([]string{"http://" + listener.Addr().String()}, false) defer ethash.Close() // Stream a lot of work task and ensure all the notifications bubble out @@ -103,7 +103,7 @@ func TestRemoteMultiNotify(t *testing.T) { header := &types.Header{Number: big.NewInt(int64(i)), Difficulty: big.NewInt(100)} block := types.NewBlockWithHeader(header) - ethash.Seal(nil, block, nil) + ethash.Seal(nil, block, nil, nil) } for i := 0; i < cap(sink); i++ { select { @@ -113,3 +113,87 @@ func TestRemoteMultiNotify(t *testing.T) { } } } + +// Tests whether stale solutions are correctly processed. +func TestStaleSubmission(t *testing.T) { + ethash := NewTester(nil, true) + defer ethash.Close() + api := &API{ethash} + + fakeNonce, fakeDigest := types.BlockNonce{0x01, 0x02, 0x03}, common.HexToHash("deadbeef") + + testcases := []struct { + headers []*types.Header + submitIndex int + submitRes bool + }{ + // Case1: submit solution for the latest mining package + { + []*types.Header{ + {ParentHash: common.BytesToHash([]byte{0xa}), Number: big.NewInt(1), Difficulty: big.NewInt(100000000)}, + }, + 0, + true, + }, + // Case2: submit solution for the previous package but have same parent. + { + []*types.Header{ + {ParentHash: common.BytesToHash([]byte{0xb}), Number: big.NewInt(2), Difficulty: big.NewInt(100000000)}, + {ParentHash: common.BytesToHash([]byte{0xb}), Number: big.NewInt(2), Difficulty: big.NewInt(100000001)}, + }, + 0, + true, + }, + // Case3: submit stale but acceptable solution + { + []*types.Header{ + {ParentHash: common.BytesToHash([]byte{0xc}), Number: big.NewInt(3), Difficulty: big.NewInt(100000000)}, + {ParentHash: common.BytesToHash([]byte{0xd}), Number: big.NewInt(9), Difficulty: big.NewInt(100000000)}, + }, + 0, + true, + }, + // Case4: submit very old solution + { + []*types.Header{ + {ParentHash: common.BytesToHash([]byte{0xe}), Number: big.NewInt(10), Difficulty: big.NewInt(100000000)}, + {ParentHash: common.BytesToHash([]byte{0xf}), Number: big.NewInt(17), Difficulty: big.NewInt(100000000)}, + }, + 0, + false, + }, + } + results := make(chan *types.Block, 16) + + for id, c := range testcases { + for _, h := range c.headers { + ethash.Seal(nil, types.NewBlockWithHeader(h), results, nil) + } + if res := api.SubmitWork(fakeNonce, ethash.SealHash(c.headers[c.submitIndex]), fakeDigest); res != c.submitRes { + t.Errorf("case %d submit result mismatch, want %t, get %t", id+1, c.submitRes, res) + } + if !c.submitRes { + continue + } + select { + case res := <-results: + if res.Header().Nonce != fakeNonce { + t.Errorf("case %d block nonce mismatch, want %s, get %s", id+1, fakeNonce, res.Header().Nonce) + } + if res.Header().MixDigest != fakeDigest { + t.Errorf("case %d block digest mismatch, want %s, get %s", id+1, fakeDigest, res.Header().MixDigest) + } + if res.Header().Difficulty.Uint64() != c.headers[c.submitIndex].Difficulty.Uint64() { + t.Errorf("case %d block difficulty mismatch, want %d, get %d", id+1, c.headers[c.submitIndex].Difficulty, res.Header().Difficulty) + } + if res.Header().Number.Uint64() != c.headers[c.submitIndex].Number.Uint64() { + t.Errorf("case %d block number mismatch, want %d, get %d", id+1, c.headers[c.submitIndex].Number.Uint64(), res.Header().Number.Uint64()) + } + if res.Header().ParentHash != c.headers[c.submitIndex].ParentHash { + t.Errorf("case %d block parent hash mismatch, want %s, get %s", id+1, c.headers[c.submitIndex].ParentHash.Hex(), res.Header().ParentHash.Hex()) + } + case <-time.NewTimer(time.Second).C: + t.Errorf("case %d fetch ethash result timeout", id+1) + } + } +} diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 704a6cdbaa..0a8b9a9942 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -294,7 +294,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl failed = err break } - // Reference the trie twice, once for us, once for the trancer + // Reference the trie twice, once for us, once for the tracer database.TrieDB().Reference(root, common.Hash{}) if number >= origin { database.TrieDB().Reference(root, common.Hash{}) diff --git a/eth/backend.go b/eth/backend.go index da7e0b2cd5..3032e1a6d3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -130,7 +130,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { chainConfig: chainConfig, eventMux: ctx.EventMux, accountManager: ctx.AccountManager, - engine: CreateConsensusEngine(ctx, chainConfig, &config.Ethash, config.MinerNotify, chainDb), + engine: CreateConsensusEngine(ctx, chainConfig, &config.Ethash, config.MinerNotify, config.MinerNoverify, chainDb), shutdownChan: make(chan bool), networkID: config.NetworkId, gasPrice: config.MinerGasPrice, @@ -216,7 +216,7 @@ func CreateDB(ctx *node.ServiceContext, config *Config, name string) (ethdb.Data } // CreateConsensusEngine creates the required type of consensus engine instance for an Ethereum service -func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, db ethdb.Database) consensus.Engine { +func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainConfig, config *ethash.Config, notify []string, noverify bool, db ethdb.Database) consensus.Engine { // If proof-of-authority is requested, set it up if chainConfig.Clique != nil { return clique.New(chainConfig.Clique, db) @@ -228,7 +228,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainCo return ethash.NewFaker() case ethash.ModeTest: log.Warn("Ethash used in test mode") - return ethash.NewTester(nil) + return ethash.NewTester(nil, noverify) case ethash.ModeShared: log.Warn("Ethash used in shared mode") return ethash.NewShared() @@ -240,7 +240,7 @@ func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainCo DatasetDir: config.DatasetDir, DatasetsInMem: config.DatasetsInMem, DatasetsOnDisk: config.DatasetsOnDisk, - }, notify) + }, notify, noverify) engine.SetThreads(-1) // Disable CPU mining return engine } diff --git a/eth/config.go b/eth/config.go index 5b2eca585c..517d7d8f3d 100644 --- a/eth/config.go +++ b/eth/config.go @@ -101,6 +101,7 @@ type Config struct { MinerExtraData []byte `toml:",omitempty"` MinerGasPrice *big.Int MinerRecommit time.Duration + MinerNoverify bool // Ethash options Ethash ethash.Config diff --git a/eth/gen_config.go b/eth/gen_config.go index a72e35bcd1..df4ffeb11f 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -35,6 +35,7 @@ func (c Config) MarshalTOML() (interface{}, error) { MinerExtraData hexutil.Bytes `toml:",omitempty"` MinerGasPrice *big.Int MinerRecommit time.Duration + MinerNoverify bool Ethash ethash.Config TxPool core.TxPoolConfig GPO gasprice.Config @@ -58,6 +59,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.MinerExtraData = c.MinerExtraData enc.MinerGasPrice = c.MinerGasPrice enc.MinerRecommit = c.MinerRecommit + enc.MinerNoverify = c.MinerNoverify enc.Ethash = c.Ethash enc.TxPool = c.TxPool enc.GPO = c.GPO @@ -81,11 +83,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { TrieCache *int TrieTimeout *time.Duration Etherbase *common.Address `toml:",omitempty"` - MinerThreads *int `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData *hexutil.Bytes `toml:",omitempty"` MinerGasPrice *big.Int MinerRecommit *time.Duration + MinerNoverify *bool Ethash *ethash.Config TxPool *core.TxPoolConfig GPO *gasprice.Config @@ -144,6 +146,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.MinerRecommit != nil { c.MinerRecommit = *dec.MinerRecommit } + if dec.MinerNoverify != nil { + c.MinerNoverify = *dec.MinerNoverify + } if dec.Ethash != nil { c.Ethash = *dec.Ethash } diff --git a/les/backend.go b/les/backend.go index 75049da085..a3474a6830 100644 --- a/les/backend.go +++ b/les/backend.go @@ -102,7 +102,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { peers: peers, reqDist: newRequestDistributor(peers, quitSync), accountManager: ctx.AccountManager, - engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, chainDb), + engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, false, chainDb), shutdownChan: make(chan bool), networkId: config.NetworkId, bloomRequests: make(chan chan *bloombits.Retrieval), diff --git a/miner/worker.go b/miner/worker.go index 1a881799d5..ca68da6e94 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -73,7 +73,7 @@ const ( // increasing upper limit or decreasing lower limit so that the limit can be reachable. intervalAdjustBias = 200 * 1000.0 * 1000.0 - // staleThreshold is the maximum distance of the acceptable stale block. + // staleThreshold is the maximum depth of the acceptable stale block. staleThreshold = 7 ) @@ -139,7 +139,7 @@ type worker struct { // Channels newWorkCh chan *newWorkReq taskCh chan *task - resultCh chan *task + resultCh chan *types.Block startCh chan struct{} exitCh chan struct{} resubmitIntervalCh chan time.Duration @@ -186,7 +186,7 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, chainSideCh: make(chan core.ChainSideEvent, chainSideChanSize), newWorkCh: make(chan *newWorkReq), taskCh: make(chan *task), - resultCh: make(chan *task, resultQueueSize), + resultCh: make(chan *types.Block, resultQueueSize), exitCh: make(chan struct{}), startCh: make(chan struct{}, 1), resubmitIntervalCh: make(chan time.Duration), @@ -269,18 +269,10 @@ func (w *worker) isRunning() bool { return atomic.LoadInt32(&w.running) == 1 } -// close terminates all background threads maintained by the worker and cleans up buffered channels. +// close terminates all background threads maintained by the worker. // Note the worker does not support being closed multiple times. func (w *worker) close() { close(w.exitCh) - // Clean up buffered channels - for empty := false; !empty; { - select { - case <-w.resultCh: - default: - empty = true - } - } } // newWorkLoop is a standalone goroutine to submit new mining work upon received events. @@ -471,42 +463,6 @@ func (w *worker) mainLoop() { } } -// seal pushes a sealing task to consensus engine and submits the result. -func (w *worker) seal(t *task, stop <-chan struct{}) { - if w.skipSealHook != nil && w.skipSealHook(t) { - return - } - // The reason for caching task first is: - // A previous sealing action will be canceled by subsequent actions, - // however, remote miner may submit a result based on the cancelled task. - // So we should only submit the pending state corresponding to the seal result. - // TODO(rjl493456442) Replace the seal-wait logic structure - w.pendingMu.Lock() - w.pendingTasks[w.engine.SealHash(t.block.Header())] = t - w.pendingMu.Unlock() - - if block, err := w.engine.Seal(w.chain, t.block, stop); block != nil { - sealhash := w.engine.SealHash(block.Header()) - w.pendingMu.RLock() - task, exist := w.pendingTasks[sealhash] - w.pendingMu.RUnlock() - if !exist { - log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", block.Hash()) - return - } - // Assemble sealing result - task.block = block - log.Info("Successfully sealed new block", "number", block.Number(), "sealhash", sealhash, "hash", block.Hash(), - "elapsed", common.PrettyDuration(time.Since(task.createdAt))) - select { - case w.resultCh <- task: - case <-w.exitCh: - } - } else if err != nil { - log.Warn("Block sealing failed", "err", err) - } -} - // taskLoop is a standalone goroutine to fetch sealing task from the generator and // push them to consensus engine. func (w *worker) taskLoop() { @@ -533,10 +489,20 @@ func (w *worker) taskLoop() { if sealHash == prev { continue } + // Interrupt previous sealing operation interrupt() - stopCh = make(chan struct{}) - prev = sealHash - go w.seal(task, stopCh) + stopCh, prev = make(chan struct{}), sealHash + + if w.skipSealHook != nil && w.skipSealHook(task) { + continue + } + w.pendingMu.Lock() + w.pendingTasks[w.engine.SealHash(task.block.Header())] = task + w.pendingMu.Unlock() + + if err := w.engine.Seal(w.chain, task.block, w.resultCh, stopCh); err != nil { + log.Warn("Block sealing failed", "err", err) + } case <-w.exitCh: interrupt() return @@ -549,38 +515,54 @@ func (w *worker) taskLoop() { func (w *worker) resultLoop() { for { select { - case result := <-w.resultCh: + case block := <-w.resultCh: // Short circuit when receiving empty result. - if result == nil { + if block == nil { continue } // Short circuit when receiving duplicate result caused by resubmitting. - block := result.block if w.chain.HasBlock(block.Hash(), block.NumberU64()) { continue } - // Update the block hash in all logs since it is now available and not when the - // receipt/log of individual transactions were created. - for _, r := range result.receipts { - for _, l := range r.Logs { - l.BlockHash = block.Hash() - } + var ( + sealhash = w.engine.SealHash(block.Header()) + hash = block.Hash() + ) + w.pendingMu.RLock() + task, exist := w.pendingTasks[sealhash] + w.pendingMu.RUnlock() + if !exist { + log.Error("Block found but no relative pending task", "number", block.Number(), "sealhash", sealhash, "hash", hash) + continue } - for _, log := range result.state.Logs() { - log.BlockHash = block.Hash() + // Different block could share same sealhash, deep copy here to prevent write-write conflict. + var ( + receipts = make([]*types.Receipt, len(task.receipts)) + logs []*types.Log + ) + for i, receipt := range task.receipts { + receipts[i] = new(types.Receipt) + *receipts[i] = *receipt + // Update the block hash in all logs since it is now available and not when the + // receipt/log of individual transactions were created. + for _, log := range receipt.Logs { + log.BlockHash = hash + } + logs = append(logs, receipt.Logs...) } // Commit block and state to database. - stat, err := w.chain.WriteBlockWithState(block, result.receipts, result.state) + stat, err := w.chain.WriteBlockWithState(block, receipts, task.state) if err != nil { log.Error("Failed writing block to chain", "err", err) continue } + log.Info("Successfully sealed new block", "number", block.Number(), "sealhash", sealhash, "hash", hash, + "elapsed", common.PrettyDuration(time.Since(task.createdAt))) + // Broadcast the block and announce chain insertion event w.mux.Post(core.NewMinedBlockEvent{Block: block}) - var ( - events []interface{} - logs = result.state.Logs() - ) + + var events []interface{} switch stat { case core.CanonStatTy: events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) From e8f229b82ef99213f8f84b8a71f752b236024494 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 29 Aug 2018 12:21:12 +0300 Subject: [PATCH 64/96] cmd, core, eth, miner, params: configurable gas floor and ceil --- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 1 + cmd/puppeth/module_node.go | 11 +++++++++-- cmd/puppeth/wizard_node.go | 6 +++++- cmd/utils/flags.go | 27 ++++++++++++++++----------- core/bench_test.go | 3 ++- core/block_validator.go | 22 ++++++++++++++-------- core/chain_makers.go | 2 +- eth/backend.go | 2 +- eth/config.go | 6 +++++- eth/gasprice/gasprice.go | 2 +- eth/gen_config.go | 12 ++++++++++++ internal/ethapi/api.go | 2 +- miner/miner.go | 4 ++-- miner/stress_clique.go | 2 ++ miner/stress_ethash.go | 2 ++ miner/worker.go | 9 +++++++-- miner/worker_test.go | 2 +- params/denomination.go | 16 +++++----------- params/protocol_params.go | 4 ---- 20 files changed, 88 insertions(+), 49 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 0ed67d0d57..4d7e98698c 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -101,6 +101,7 @@ var ( utils.MinerNotifyFlag, utils.MinerGasTargetFlag, utils.MinerLegacyGasTargetFlag, + utils.MinerGasLimitFlag, utils.MinerGasPriceFlag, utils.MinerLegacyGasPriceFlag, utils.MinerEtherbaseFlag, @@ -236,7 +237,6 @@ func init() { // Start system runtime metrics collection go metrics.CollectProcessMetrics(3 * time.Second) - utils.SetupNetwork(ctx) return nil } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 2f8260e0f6..a674eca4f1 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -189,6 +189,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.MinerNotifyFlag, utils.MinerGasPriceFlag, utils.MinerGasTargetFlag, + utils.MinerGasLimitFlag, utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 8ad41555e1..038152a3e4 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -42,7 +42,7 @@ ADD genesis.json /genesis.json RUN \ echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}} echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}} - echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gasprice {{.GasPrice}}' >> geth.sh + echo $'exec geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--miner.etherbase {{.Etherbase}} --mine --miner.threads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --miner.gastarget {{.GasTarget}} --miner.gaslimit {{.GasLimit}} --miner.gasprice {{.GasPrice}}' >> geth.sh ENTRYPOINT ["/bin/sh", "geth.sh"] ` @@ -68,6 +68,7 @@ services: - STATS_NAME={{.Ethstats}} - MINER_NAME={{.Etherbase}} - GAS_TARGET={{.GasTarget}} + - GAS_LIMIT={{.GasLimit}} - GAS_PRICE={{.GasPrice}} logging: driver: "json-file" @@ -104,6 +105,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "Ethstats": config.ethstats, "Etherbase": config.etherbase, "GasTarget": uint64(1000000 * config.gasTarget), + "GasLimit": uint64(1000000 * config.gasLimit), "GasPrice": uint64(1000000000 * config.gasPrice), "Unlock": config.keyJSON != "", }) @@ -122,6 +124,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")], "Etherbase": config.etherbase, "GasTarget": config.gasTarget, + "GasLimit": config.gasLimit, "GasPrice": config.gasPrice, }) files[filepath.Join(workdir, "docker-compose.yaml")] = composefile.Bytes() @@ -160,6 +163,7 @@ type nodeInfos struct { keyJSON string keyPass string gasTarget float64 + gasLimit float64 gasPrice float64 } @@ -175,8 +179,9 @@ func (info *nodeInfos) Report() map[string]string { } if info.gasTarget > 0 { // Miner or signer node - report["Gas limit (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget) report["Gas price (minimum accepted)"] = fmt.Sprintf("%0.3f GWei", info.gasPrice) + report["Gas floor (baseline target)"] = fmt.Sprintf("%0.3f MGas", info.gasTarget) + report["Gas ceil (target maximum)"] = fmt.Sprintf("%0.3f MGas", info.gasLimit) if info.etherbase != "" { // Ethash proof-of-work miner @@ -217,6 +222,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) totalPeers, _ := strconv.Atoi(infos.envvars["TOTAL_PEERS"]) lightPeers, _ := strconv.Atoi(infos.envvars["LIGHT_PEERS"]) gasTarget, _ := strconv.ParseFloat(infos.envvars["GAS_TARGET"], 64) + gasLimit, _ := strconv.ParseFloat(infos.envvars["GAS_LIMIT"], 64) gasPrice, _ := strconv.ParseFloat(infos.envvars["GAS_PRICE"], 64) // Container available, retrieve its node ID and its genesis json @@ -256,6 +262,7 @@ func checkNode(client *sshClient, network string, boot bool) (*nodeInfos, error) keyJSON: keyJSON, keyPass: keyPass, gasTarget: gasTarget, + gasLimit: gasLimit, gasPrice: gasPrice, } stats.enode = fmt.Sprintf("enode://%s@%s:%d", id, client.address, stats.port) diff --git a/cmd/puppeth/wizard_node.go b/cmd/puppeth/wizard_node.go index 9b22da4460..49b10a023a 100644 --- a/cmd/puppeth/wizard_node.go +++ b/cmd/puppeth/wizard_node.go @@ -50,7 +50,7 @@ func (w *wizard) deployNode(boot bool) { if boot { infos = &nodeInfos{port: 30303, peersTotal: 512, peersLight: 256} } else { - infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 4.7, gasPrice: 18} + infos = &nodeInfos{port: 30303, peersTotal: 50, peersLight: 0, gasTarget: 7.5, gasLimit: 10, gasPrice: 1} } } existed := err == nil @@ -152,6 +152,10 @@ func (w *wizard) deployNode(boot bool) { fmt.Printf("What gas limit should empty blocks target (MGas)? (default = %0.3f)\n", infos.gasTarget) infos.gasTarget = w.readDefaultFloat(infos.gasTarget) + fmt.Println() + fmt.Printf("What gas limit should full blocks target (MGas)? (default = %0.3f)\n", infos.gasLimit) + infos.gasLimit = w.readDefaultFloat(infos.gasLimit) + fmt.Println() fmt.Printf("What gas price should the signer require (GWei)? (default = %0.3f)\n", infos.gasPrice) infos.gasPrice = w.readDefaultFloat(infos.gasPrice) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 13430ad565..495bfe13e0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -329,12 +329,17 @@ var ( MinerGasTargetFlag = cli.Uint64Flag{ Name: "miner.gastarget", Usage: "Target gas floor for mined blocks", - Value: params.GenesisGasLimit, + Value: eth.DefaultConfig.MinerGasFloor, } MinerLegacyGasTargetFlag = cli.Uint64Flag{ Name: "targetgaslimit", Usage: "Target gas floor for mined blocks (deprecated, use --miner.gastarget)", - Value: params.GenesisGasLimit, + Value: eth.DefaultConfig.MinerGasFloor, + } + MinerGasLimitFlag = cli.Uint64Flag{ + Name: "miner.gaslimit", + Usage: "Target gas ceiling for mined blocks", + Value: eth.DefaultConfig.MinerGasCeil, } MinerGasPriceFlag = BigFlag{ Name: "miner.gasprice", @@ -1146,6 +1151,15 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(MinerExtraDataFlag.Name) { cfg.MinerExtraData = []byte(ctx.GlobalString(MinerExtraDataFlag.Name)) } + if ctx.GlobalIsSet(MinerLegacyGasTargetFlag.Name) { + cfg.MinerGasFloor = ctx.GlobalUint64(MinerLegacyGasTargetFlag.Name) + } + if ctx.GlobalIsSet(MinerGasTargetFlag.Name) { + cfg.MinerGasFloor = ctx.GlobalUint64(MinerGasTargetFlag.Name) + } + if ctx.GlobalIsSet(MinerGasLimitFlag.Name) { + cfg.MinerGasCeil = ctx.GlobalUint64(MinerGasLimitFlag.Name) + } if ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { cfg.MinerGasPrice = GlobalBig(ctx, MinerLegacyGasPriceFlag.Name) } @@ -1270,15 +1284,6 @@ func RegisterEthStatsService(stack *node.Node, url string) { } } -// SetupNetwork configures the system for either the main net or some test network. -func SetupNetwork(ctx *cli.Context) { - // TODO(fjl): move target gas limit into config - params.TargetGasLimit = ctx.GlobalUint64(MinerLegacyGasTargetFlag.Name) - if ctx.GlobalIsSet(MinerGasTargetFlag.Name) { - params.TargetGasLimit = ctx.GlobalUint64(MinerGasTargetFlag.Name) - } -} - func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/core/bench_test.go b/core/bench_test.go index 748aebe407..8d95456e9f 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -111,7 +111,8 @@ func init() { func genTxRing(naccounts int) func(int, *BlockGen) { from := 0 return func(i int, gen *BlockGen) { - gas := CalcGasLimit(gen.PrevBlock(i - 1)) + block := gen.PrevBlock(i - 1) + gas := CalcGasLimit(block, block.GasLimit(), block.GasLimit()) for { gas -= params.TxGas if gas < params.TxGas { diff --git a/core/block_validator.go b/core/block_validator.go index ecd6a89bc2..1329f62427 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -101,9 +101,11 @@ func (v *BlockValidator) ValidateState(block, parent *types.Block, statedb *stat return nil } -// CalcGasLimit computes the gas limit of the next block after parent. -// This is miner strategy, not consensus protocol. -func CalcGasLimit(parent *types.Block) uint64 { +// CalcGasLimit computes the gas limit of the next block after parent. It aims +// to keep the baseline gas above the provided floor, and increase it towards the +// ceil if the blocks are full. If the ceil is exceeded, it will always decrease +// the gas allowance. +func CalcGasLimit(parent *types.Block, gasFloor, gasCeil uint64) uint64 { // contrib = (parentGasUsed * 3 / 2) / 1024 contrib := (parent.GasUsed() + parent.GasUsed()/2) / params.GasLimitBoundDivisor @@ -121,12 +123,16 @@ func CalcGasLimit(parent *types.Block) uint64 { if limit < params.MinGasLimit { limit = params.MinGasLimit } - // however, if we're now below the target (TargetGasLimit) we increase the - // limit as much as we can (parentGasLimit / 1024 -1) - if limit < params.TargetGasLimit { + // If we're outside our allowed gas range, we try to hone towards them + if limit < gasFloor { limit = parent.GasLimit() + decay - if limit > params.TargetGasLimit { - limit = params.TargetGasLimit + if limit > gasFloor { + limit = gasFloor + } + } else if limit > gasCeil { + limit = parent.GasLimit() - decay + if limit < gasCeil { + limit = gasCeil } } return limit diff --git a/core/chain_makers.go b/core/chain_makers.go index c1e4b4264a..de0fc6be9e 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -240,7 +240,7 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S Difficulty: parent.Difficulty(), UncleHash: parent.UncleHash(), }), - GasLimit: CalcGasLimit(parent), + GasLimit: CalcGasLimit(parent, parent.GasLimit(), parent.GasLimit()), Number: new(big.Int).Add(parent.Number(), common.Big1), Time: time, } diff --git a/eth/backend.go b/eth/backend.go index 3032e1a6d3..9926225f24 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -173,7 +173,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { return nil, err } - eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine, config.MinerRecommit) + eth.miner = miner.New(eth, eth.chainConfig, eth.EventMux(), eth.engine, config.MinerRecommit, config.MinerGasFloor, config.MinerGasCeil) eth.miner.SetExtra(makeExtraData(config.MinerExtraData)) eth.APIBackend = &EthAPIBackend{eth, nil} diff --git a/eth/config.go b/eth/config.go index 517d7d8f3d..f1a402e370 100644 --- a/eth/config.go +++ b/eth/config.go @@ -48,7 +48,9 @@ var DefaultConfig = Config{ DatabaseCache: 768, TrieCache: 256, TrieTimeout: 60 * time.Minute, - MinerGasPrice: big.NewInt(18 * params.Shannon), + MinerGasFloor: 8000000, + MinerGasCeil: 8000000, + MinerGasPrice: big.NewInt(params.GWei), MinerRecommit: 3 * time.Second, TxPool: core.DefaultTxPoolConfig, @@ -99,6 +101,8 @@ type Config struct { Etherbase common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData []byte `toml:",omitempty"` + MinerGasFloor uint64 + MinerGasCeil uint64 MinerGasPrice *big.Int MinerRecommit time.Duration MinerNoverify bool diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 54325692c6..3b8db78a1c 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -var maxPrice = big.NewInt(500 * params.Shannon) +var maxPrice = big.NewInt(500 * params.GWei) type Config struct { Blocks int diff --git a/eth/gen_config.go b/eth/gen_config.go index df4ffeb11f..d401a917d2 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -33,6 +33,8 @@ func (c Config) MarshalTOML() (interface{}, error) { Etherbase common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData hexutil.Bytes `toml:",omitempty"` + MinerGasFloor uint64 + MinerGasCeil uint64 MinerGasPrice *big.Int MinerRecommit time.Duration MinerNoverify bool @@ -57,6 +59,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.Etherbase = c.Etherbase enc.MinerNotify = c.MinerNotify enc.MinerExtraData = c.MinerExtraData + enc.MinerGasFloor = c.MinerGasFloor + enc.MinerGasCeil = c.MinerGasCeil enc.MinerGasPrice = c.MinerGasPrice enc.MinerRecommit = c.MinerRecommit enc.MinerNoverify = c.MinerNoverify @@ -85,6 +89,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { Etherbase *common.Address `toml:",omitempty"` MinerNotify []string `toml:",omitempty"` MinerExtraData *hexutil.Bytes `toml:",omitempty"` + MinerGasFloor *uint64 + MinerGasCeil *uint64 MinerGasPrice *big.Int MinerRecommit *time.Duration MinerNoverify *bool @@ -140,6 +146,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.MinerExtraData != nil { c.MinerExtraData = *dec.MinerExtraData } + if dec.MinerGasFloor != nil { + c.MinerGasFloor = *dec.MinerGasFloor + } + if dec.MinerGasCeil != nil { + c.MinerGasCeil = *dec.MinerGasCeil + } if dec.MinerGasPrice != nil { c.MinerGasPrice = dec.MinerGasPrice } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a465a59046..f7379e228c 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -47,7 +47,7 @@ import ( ) const ( - defaultGasPrice = 50 * params.Shannon + defaultGasPrice = params.GWei ) // PublicEthereumAPI provides an API to access Ethereum related information. diff --git a/miner/miner.go b/miner/miner.go index c5a0c9d62a..7f194db261 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -52,13 +52,13 @@ type Miner struct { shouldStart int32 // should start indicates whether we should start after sync } -func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, recommit time.Duration) *Miner { +func New(eth Backend, config *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, recommit time.Duration, gasFloor, gasCeil uint64) *Miner { miner := &Miner{ eth: eth, mux: mux, engine: engine, exitCh: make(chan struct{}), - worker: newWorker(config, engine, eth, mux, recommit), + worker: newWorker(config, engine, eth, mux, recommit, gasFloor, gasCeil), canStart: 1, } go miner.update() diff --git a/miner/stress_clique.go b/miner/stress_clique.go index 8545b9413a..8961091d56 100644 --- a/miner/stress_clique.go +++ b/miner/stress_clique.go @@ -206,6 +206,8 @@ func makeSealer(genesis *core.Genesis, nodes []string) (*node.Node, error) { DatabaseHandles: 256, TxPool: core.DefaultTxPoolConfig, GPO: eth.DefaultConfig.GPO, + MinerGasFloor: genesis.GasLimit * 9 / 10, + MinerGasCeil: genesis.GasLimit * 11 / 10, MinerGasPrice: big.NewInt(1), MinerRecommit: time.Second, }) diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go index 4ed8aeee4f..5ed11d73a5 100644 --- a/miner/stress_ethash.go +++ b/miner/stress_ethash.go @@ -186,6 +186,8 @@ func makeMiner(genesis *core.Genesis, nodes []string) (*node.Node, error) { TxPool: core.DefaultTxPoolConfig, GPO: eth.DefaultConfig.GPO, Ethash: eth.DefaultConfig.Ethash, + MinerGasFloor: genesis.GasLimit * 9 / 10, + MinerGasCeil: genesis.GasLimit * 11 / 10, MinerGasPrice: big.NewInt(1), MinerRecommit: time.Second, }) diff --git a/miner/worker.go b/miner/worker.go index ca68da6e94..5348cb3f1e 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -127,6 +127,9 @@ type worker struct { eth Backend chain *core.BlockChain + gasFloor uint64 + gasCeil uint64 + // Subscriptions mux *event.TypeMux txsCh chan core.NewTxsEvent @@ -171,13 +174,15 @@ type worker struct { resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. } -func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, recommit time.Duration) *worker { +func newWorker(config *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, recommit time.Duration, gasFloor, gasCeil uint64) *worker { worker := &worker{ config: config, engine: engine, eth: eth, mux: mux, chain: eth.BlockChain(), + gasFloor: gasFloor, + gasCeil: gasCeil, possibleUncles: make(map[common.Hash]*types.Block), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), pendingTasks: make(map[common.Hash]*task), @@ -807,7 +812,7 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool) { header := &types.Header{ ParentHash: parent.Hash(), Number: num.Add(num, common.Big1), - GasLimit: core.CalcGasLimit(parent), + GasLimit: core.CalcGasLimit(parent, w.gasFloor, w.gasCeil), Extra: w.extra, Time: big.NewInt(tstamp), } diff --git a/miner/worker_test.go b/miner/worker_test.go index 16708c18c6..9f9a3b4a5e 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -119,7 +119,7 @@ func (b *testWorkerBackend) PostChainEvents(events []interface{}) { func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) (*worker, *testWorkerBackend) { backend := newTestWorkerBackend(t, chainConfig, engine) backend.txPool.AddLocals(pendingTxs) - w := newWorker(chainConfig, engine, backend, new(event.TypeMux), time.Second) + w := newWorker(chainConfig, engine, backend, new(event.TypeMux), time.Second, params.GenesisGasLimit, params.GenesisGasLimit) w.setEtherbase(testBankAddress) return w, backend } diff --git a/params/denomination.go b/params/denomination.go index 1a309827da..fb4da7f412 100644 --- a/params/denomination.go +++ b/params/denomination.go @@ -17,18 +17,12 @@ package params // These are the multipliers for ether denominations. -// Example: To get the wei value of an amount in 'douglas', use +// Example: To get the wei value of an amount in 'gwei', use // -// new(big.Int).Mul(value, big.NewInt(params.Douglas)) +// new(big.Int).Mul(value, big.NewInt(params.GWei)) // const ( - Wei = 1 - Ada = 1e3 - Babbage = 1e6 - Shannon = 1e9 - Szabo = 1e12 - Finney = 1e15 - Ether = 1e18 - Einstein = 1e21 - Douglas = 1e42 + Wei = 1 + GWei = 1e9 + Ether = 1e18 ) diff --git a/params/protocol_params.go b/params/protocol_params.go index 46624ac9a2..4b53b3320a 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -18,10 +18,6 @@ package params import "math/big" -var ( - TargetGasLimit = GenesisGasLimit // The artificial target -) - const ( GasLimitBoundDivisor uint64 = 1024 // The bound divisor of the gas limit, used in update calculations. MinGasLimit uint64 = 5000 // Minimum the gas limit may ever be. From f751c6ed47db49e0032dfcd0f4c06ce72f3616bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 28 Aug 2018 16:48:20 +0300 Subject: [PATCH 65/96] miner: track uncles more aggressively --- miner/worker.go | 32 ++++++++--------- miner/worker_test.go | 82 ++++++++++++++++++++++++++------------------ 2 files changed, 63 insertions(+), 51 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index ca68da6e94..99f1f8dcf6 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -18,7 +18,7 @@ package miner import ( "bytes" - "fmt" + "errors" "math/big" "sync" "sync/atomic" @@ -615,13 +615,16 @@ func (w *worker) makeCurrent(parent *types.Block, header *types.Header) error { func (w *worker) commitUncle(env *environment, uncle *types.Header) error { hash := uncle.Hash() if env.uncles.Contains(hash) { - return fmt.Errorf("uncle not unique") + return errors.New("uncle not unique") + } + if env.header.ParentHash == uncle.ParentHash { + return errors.New("uncle is sibling") } if !env.ancestors.Contains(uncle.ParentHash) { - return fmt.Errorf("uncle's parent unknown (%x)", uncle.ParentHash[0:4]) + return errors.New("uncle's parent unknown") } if env.family.Contains(hash) { - return fmt.Errorf("uncle already in family (%x)", hash) + return errors.New("uncle already included") } env.uncles.Add(uncle.Hash()) return nil @@ -847,29 +850,24 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool) { if w.config.DAOForkSupport && w.config.DAOForkBlock != nil && w.config.DAOForkBlock.Cmp(header.Number) == 0 { misc.ApplyDAOHardFork(env.state) } - - // compute uncles for the new block. - var ( - uncles []*types.Header - badUncles []common.Hash - ) + // Accumulate the uncles for the current block + for hash, uncle := range w.possibleUncles { + if uncle.NumberU64()+staleThreshold <= header.Number.Uint64() { + delete(w.possibleUncles, hash) + } + } + uncles := make([]*types.Header, 0, 2) for hash, uncle := range w.possibleUncles { if len(uncles) == 2 { break } if err := w.commitUncle(env, uncle.Header()); err != nil { - log.Trace("Bad uncle found and will be removed", "hash", hash) - log.Trace(fmt.Sprint(uncle)) - - badUncles = append(badUncles, hash) + log.Trace("Possible uncle rejected", "hash", hash, "reason", err) } else { log.Debug("Committing new uncle to block", "hash", hash) uncles = append(uncles, uncle.Header()) } } - for _, hash := range badUncles { - delete(w.possibleUncles, hash) - } if !noempty { // Create an empty block based on temporary copied state for sealing in advance without waiting block diff --git a/miner/worker_test.go b/miner/worker_test.go index 16708c18c6..40a57d1376 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -45,8 +45,8 @@ var ( testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) testBankFunds = big.NewInt(1000000000000000000) - acc1Key, _ = crypto.GenerateKey() - acc1Addr = crypto.PubkeyToAddress(acc1Key.PublicKey) + testUserKey, _ = crypto.GenerateKey() + testUserAddress = crypto.PubkeyToAddress(testUserKey.PublicKey) // Test transactions pendingTxs []*types.Transaction @@ -62,9 +62,9 @@ func init() { Period: 10, Epoch: 30000, } - tx1, _ := types.SignTx(types.NewTransaction(0, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + tx1, _ := types.SignTx(types.NewTransaction(0, testUserAddress, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) pendingTxs = append(pendingTxs, tx1) - tx2, _ := types.SignTx(types.NewTransaction(1, acc1Addr, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) + tx2, _ := types.SignTx(types.NewTransaction(1, testUserAddress, big.NewInt(1000), params.TxGas, nil, nil), types.HomesteadSigner{}, testBankKey) newTxs = append(newTxs, tx2) } @@ -77,7 +77,7 @@ type testWorkerBackend struct { uncleBlock *types.Block } -func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) *testWorkerBackend { +func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, n int) *testWorkerBackend { var ( db = ethdb.NewMemDatabase() gspec = core.Genesis{ @@ -92,14 +92,28 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine copy(gspec.ExtraData[32:], testBankAddress[:]) case *ethash.Ethash: default: - t.Fatal("unexpect consensus engine type") + t.Fatalf("unexpected consensus engine type: %T", engine) } genesis := gspec.MustCommit(db) chain, _ := core.NewBlockChain(db, nil, gspec.Config, engine, vm.Config{}) txpool := core.NewTxPool(testTxPoolConfig, chainConfig, chain) - blocks, _ := core.GenerateChain(chainConfig, genesis, engine, db, 1, func(i int, gen *core.BlockGen) { - gen.SetCoinbase(acc1Addr) + + // Generate a small n-block chain and an uncle block for it + if n > 0 { + blocks, _ := core.GenerateChain(chainConfig, genesis, engine, db, n, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(testBankAddress) + }) + if _, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("failed to insert origin chain: %v", err) + } + } + parent := genesis + if n > 0 { + parent = chain.GetBlockByHash(chain.CurrentBlock().ParentHash()) + } + blocks, _ := core.GenerateChain(chainConfig, parent, engine, db, 1, func(i int, gen *core.BlockGen) { + gen.SetCoinbase(testUserAddress) }) return &testWorkerBackend{ @@ -116,8 +130,8 @@ func (b *testWorkerBackend) PostChainEvents(events []interface{}) { b.chain.PostChainEvents(events, nil) } -func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) (*worker, *testWorkerBackend) { - backend := newTestWorkerBackend(t, chainConfig, engine) +func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, blocks int) (*worker, *testWorkerBackend) { + backend := newTestWorkerBackend(t, chainConfig, engine, blocks) backend.txPool.AddLocals(pendingTxs) w := newWorker(chainConfig, engine, backend, new(event.TypeMux), time.Second) w.setEtherbase(testBankAddress) @@ -134,24 +148,24 @@ func TestPendingStateAndBlockClique(t *testing.T) { func testPendingStateAndBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { defer engine.Close() - w, b := newTestWorker(t, chainConfig, engine) + w, b := newTestWorker(t, chainConfig, engine, 0) defer w.close() // Ensure snapshot has been updated. time.Sleep(100 * time.Millisecond) block, state := w.pending() if block.NumberU64() != 1 { - t.Errorf("block number mismatch, has %d, want %d", block.NumberU64(), 1) + t.Errorf("block number mismatch: have %d, want %d", block.NumberU64(), 1) } - if balance := state.GetBalance(acc1Addr); balance.Cmp(big.NewInt(1000)) != 0 { - t.Errorf("account balance mismatch, has %d, want %d", balance, 1000) + if balance := state.GetBalance(testUserAddress); balance.Cmp(big.NewInt(1000)) != 0 { + t.Errorf("account balance mismatch: have %d, want %d", balance, 1000) } b.txPool.AddLocals(newTxs) // Ensure the new tx events has been processed time.Sleep(100 * time.Millisecond) block, state = w.pending() - if balance := state.GetBalance(acc1Addr); balance.Cmp(big.NewInt(2000)) != 0 { - t.Errorf("account balance mismatch, has %d, want %d", balance, 2000) + if balance := state.GetBalance(testUserAddress); balance.Cmp(big.NewInt(2000)) != 0 { + t.Errorf("account balance mismatch: have %d, want %d", balance, 2000) } } @@ -165,7 +179,7 @@ func TestEmptyWorkClique(t *testing.T) { func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { defer engine.Close() - w, _ := newTestWorker(t, chainConfig, engine) + w, _ := newTestWorker(t, chainConfig, engine, 0) defer w.close() var ( @@ -179,10 +193,10 @@ func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consens receiptLen, balance = 1, big.NewInt(1000) } if len(task.receipts) != receiptLen { - t.Errorf("receipt number mismatch has %d, want %d", len(task.receipts), receiptLen) + t.Errorf("receipt number mismatch: have %d, want %d", len(task.receipts), receiptLen) } - if task.state.GetBalance(acc1Addr).Cmp(balance) != 0 { - t.Errorf("account balance mismatch has %d, want %d", task.state.GetBalance(acc1Addr), balance) + if task.state.GetBalance(testUserAddress).Cmp(balance) != 0 { + t.Errorf("account balance mismatch: have %d, want %d", task.state.GetBalance(testUserAddress), balance) } } @@ -219,19 +233,19 @@ func TestStreamUncleBlock(t *testing.T) { ethash := ethash.NewFaker() defer ethash.Close() - w, b := newTestWorker(t, ethashChainConfig, ethash) + w, b := newTestWorker(t, ethashChainConfig, ethash, 1) defer w.close() var taskCh = make(chan struct{}) taskIndex := 0 w.newTaskHook = func(task *task) { - if task.block.NumberU64() == 1 { + if task.block.NumberU64() == 2 { if taskIndex == 2 { - has := task.block.Header().UncleHash + have := task.block.Header().UncleHash want := types.CalcUncleHash([]*types.Header{b.uncleBlock.Header()}) - if has != want { - t.Errorf("uncle hash mismatch, has %s, want %s", has.Hex(), want.Hex()) + if have != want { + t.Errorf("uncle hash mismatch: have %s, want %s", have.Hex(), want.Hex()) } } taskCh <- struct{}{} @@ -248,12 +262,12 @@ func TestStreamUncleBlock(t *testing.T) { // Ensure worker has finished initialization for { b := w.pendingBlock() - if b != nil && b.NumberU64() == 1 { + if b != nil && b.NumberU64() == 2 { break } } - w.start() + // Ignore the first two works for i := 0; i < 2; i += 1 { select { @@ -282,7 +296,7 @@ func TestRegenerateMiningBlockClique(t *testing.T) { func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { defer engine.Close() - w, b := newTestWorker(t, chainConfig, engine) + w, b := newTestWorker(t, chainConfig, engine, 0) defer w.close() var taskCh = make(chan struct{}) @@ -293,10 +307,10 @@ func testRegenerateMiningBlock(t *testing.T, chainConfig *params.ChainConfig, en if taskIndex == 2 { receiptLen, balance := 2, big.NewInt(2000) if len(task.receipts) != receiptLen { - t.Errorf("receipt number mismatch has %d, want %d", len(task.receipts), receiptLen) + t.Errorf("receipt number mismatch: have %d, want %d", len(task.receipts), receiptLen) } - if task.state.GetBalance(acc1Addr).Cmp(balance) != 0 { - t.Errorf("account balance mismatch has %d, want %d", task.state.GetBalance(acc1Addr), balance) + if task.state.GetBalance(testUserAddress).Cmp(balance) != 0 { + t.Errorf("account balance mismatch: have %d, want %d", task.state.GetBalance(testUserAddress), balance) } } taskCh <- struct{}{} @@ -347,7 +361,7 @@ func TestAdjustIntervalClique(t *testing.T) { func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine) { defer engine.Close() - w, _ := newTestWorker(t, chainConfig, engine) + w, _ := newTestWorker(t, chainConfig, engine, 0) defer w.close() w.skipSealHook = func(task *task) bool { @@ -387,10 +401,10 @@ func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine co // Check interval if minInterval != wantMinInterval { - t.Errorf("resubmit min interval mismatch want %s has %s", wantMinInterval, minInterval) + t.Errorf("resubmit min interval mismatch: have %v, want %v ", minInterval, wantMinInterval) } if recommitInterval != wantRecommitInterval { - t.Errorf("resubmit interval mismatch want %s has %s", wantRecommitInterval, recommitInterval) + t.Errorf("resubmit interval mismatch: have %v, want %v", recommitInterval, wantRecommitInterval) } result = append(result, float64(recommitInterval.Nanoseconds())) index += 1 From 957496811627d6ef33b9b4232e4d42353d376f1d Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 29 Aug 2018 12:52:21 +0200 Subject: [PATCH 66/96] cmd/swarm: disable ACT tests on windows (#17536) --- cmd/swarm/access_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index 163eb2b4d6..7a8bcf9d3d 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -13,6 +13,9 @@ // // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . + +// +build !windows + package main import ( From 75ae5af62a8dbbc4fc547d2b3b4cce192d783b2d Mon Sep 17 00:00:00 2001 From: Adam Babik Date: Wed, 29 Aug 2018 12:56:13 +0200 Subject: [PATCH 67/96] whisper: fix loop in expire() (#17532) --- whisper/whisperv5/whisper.go | 2 +- whisper/whisperv5/whisper_test.go | 33 +++++++++++++++++++------------ whisper/whisperv6/whisper.go | 2 +- whisper/whisperv6/whisper_test.go | 33 +++++++++++++++++++------------ 4 files changed, 42 insertions(+), 28 deletions(-) diff --git a/whisper/whisperv5/whisper.go b/whisper/whisperv5/whisper.go index 8e0662327a..4655458214 100644 --- a/whisper/whisperv5/whisper.go +++ b/whisper/whisperv5/whisper.go @@ -717,7 +717,7 @@ func (w *Whisper) expire() { w.stats.messagesCleared++ w.stats.memoryCleared += sz w.stats.memoryUsed -= sz - return true + return false }) w.expirations[expiry].Clear() delete(w.expirations, expiry) diff --git a/whisper/whisperv5/whisper_test.go b/whisper/whisperv5/whisper_test.go index 8af085292a..a7bd17e4d8 100644 --- a/whisper/whisperv5/whisper_test.go +++ b/whisper/whisperv5/whisper_test.go @@ -487,27 +487,34 @@ func TestExpiry(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - params.TTL = 1 - msg, err := NewSentMessage(params) - if err != nil { - t.Fatalf("failed to create new message with seed %d: %s.", seed, err) - } - env, err := msg.Wrap(params) - if err != nil { - t.Fatalf("failed Wrap with seed %d: %s.", seed, err) - } - err = w.Send(env) - if err != nil { - t.Fatalf("failed to send envelope with seed %d: %s.", seed, err) + messagesCount := 5 + + // Send a few messages one after another. Due to low PoW and expiration buckets + // with one second resolution, it covers a case when there are multiple items + // in a single expiration bucket. + for i := 0; i < messagesCount; i++ { + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } + env, err := msg.Wrap(params) + if err != nil { + t.Fatalf("failed Wrap with seed %d: %s.", seed, err) + } + + err = w.Send(env) + if err != nil { + t.Fatalf("failed to send envelope with seed %d: %s.", seed, err) + } } // wait till received or timeout var received, expired bool for j := 0; j < 20; j++ { time.Sleep(100 * time.Millisecond) - if len(w.Envelopes()) > 0 { + if len(w.Envelopes()) == messagesCount { received = true break } diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index 34c1256a6f..be633e7ff6 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -900,7 +900,7 @@ func (whisper *Whisper) expire() { whisper.stats.messagesCleared++ whisper.stats.memoryCleared += sz whisper.stats.memoryUsed -= sz - return true + return false }) whisper.expirations[expiry].Clear() delete(whisper.expirations, expiry) diff --git a/whisper/whisperv6/whisper_test.go b/whisper/whisperv6/whisper_test.go index 7fe256309e..895bb2b969 100644 --- a/whisper/whisperv6/whisper_test.go +++ b/whisper/whisperv6/whisper_test.go @@ -465,27 +465,34 @@ func TestExpiry(t *testing.T) { if err != nil { t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err) } - params.TTL = 1 - msg, err := NewSentMessage(params) - if err != nil { - t.Fatalf("failed to create new message with seed %d: %s.", seed, err) - } - env, err := msg.Wrap(params) - if err != nil { - t.Fatalf("failed Wrap with seed %d: %s.", seed, err) - } - err = w.Send(env) - if err != nil { - t.Fatalf("failed to send envelope with seed %d: %s.", seed, err) + messagesCount := 5 + + // Send a few messages one after another. Due to low PoW and expiration buckets + // with one second resolution, it covers a case when there are multiple items + // in a single expiration bucket. + for i := 0; i < messagesCount; i++ { + msg, err := NewSentMessage(params) + if err != nil { + t.Fatalf("failed to create new message with seed %d: %s.", seed, err) + } + env, err := msg.Wrap(params) + if err != nil { + t.Fatalf("failed Wrap with seed %d: %s.", seed, err) + } + + err = w.Send(env) + if err != nil { + t.Fatalf("failed to send envelope with seed %d: %s.", seed, err) + } } // wait till received or timeout var received, expired bool for j := 0; j < 20; j++ { time.Sleep(100 * time.Millisecond) - if len(w.Envelopes()) > 0 { + if len(w.Envelopes()) == messagesCount { received = true break } From f0242ee76d5d6f54fcd68732cce5c98c7c085101 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 29 Aug 2018 22:31:59 +0800 Subject: [PATCH 68/96] miner: keep the timestamp for resubmitted mining block (#17547) --- miner/worker.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index 629ce5bd24..3500ca4c23 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -111,6 +111,7 @@ const ( type newWorkReq struct { interrupt *int32 noempty bool + timestamp int64 } // intervalAdjust represents a resubmitting interval adjustment. @@ -285,6 +286,7 @@ func (w *worker) newWorkLoop(recommit time.Duration) { var ( interrupt *int32 minRecommit = recommit // minimal resubmit interval specified by user. + timestamp int64 // timestamp for each round of mining. ) timer := time.NewTimer(0) @@ -296,7 +298,7 @@ func (w *worker) newWorkLoop(recommit time.Duration) { atomic.StoreInt32(interrupt, s) } interrupt = new(int32) - w.newWorkCh <- &newWorkReq{interrupt: interrupt, noempty: noempty} + w.newWorkCh <- &newWorkReq{interrupt: interrupt, noempty: noempty, timestamp: timestamp} timer.Reset(recommit) atomic.StoreInt32(&w.newTxs, 0) } @@ -336,10 +338,12 @@ func (w *worker) newWorkLoop(recommit time.Duration) { select { case <-w.startCh: clearPending(w.chain.CurrentBlock().NumberU64()) + timestamp = time.Now().Unix() commit(false, commitInterruptNewHead) case head := <-w.chainHeadCh: clearPending(head.Block.NumberU64()) + timestamp = time.Now().Unix() commit(false, commitInterruptNewHead) case <-timer.C: @@ -398,7 +402,7 @@ func (w *worker) mainLoop() { for { select { case req := <-w.newWorkCh: - w.commitNewWork(req.interrupt, req.noempty) + w.commitNewWork(req.interrupt, req.noempty, req.timestamp) case ev := <-w.chainSideCh: if _, exist := w.possibleUncles[ev.Block.Hash()]; exist { @@ -450,7 +454,7 @@ func (w *worker) mainLoop() { } else { // If we're mining, but nothing is being processed, wake on new transactions if w.config.Clique != nil && w.config.Clique.Period == 0 { - w.commitNewWork(nil, false) + w.commitNewWork(nil, false, time.Now().Unix()) } } atomic.AddInt32(&w.newTxs, int32(len(ev.Txs))) @@ -793,20 +797,19 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin } // commitNewWork generates several new sealing tasks based on the parent block. -func (w *worker) commitNewWork(interrupt *int32, noempty bool) { +func (w *worker) commitNewWork(interrupt *int32, noempty bool, timestamp int64) { w.mu.RLock() defer w.mu.RUnlock() tstart := time.Now() parent := w.chain.CurrentBlock() - tstamp := tstart.Unix() - if parent.Time().Cmp(new(big.Int).SetInt64(tstamp)) >= 0 { - tstamp = parent.Time().Int64() + 1 + if parent.Time().Cmp(new(big.Int).SetInt64(timestamp)) >= 0 { + timestamp = parent.Time().Int64() + 1 } // this will ensure we're not going off too far in the future - if now := time.Now().Unix(); tstamp > now+1 { - wait := time.Duration(tstamp-now) * time.Second + if now := time.Now().Unix(); timestamp > now+1 { + wait := time.Duration(timestamp-now) * time.Second log.Info("Mining too far in the future", "wait", common.PrettyDuration(wait)) time.Sleep(wait) } @@ -817,7 +820,7 @@ func (w *worker) commitNewWork(interrupt *int32, noempty bool) { Number: num.Add(num, common.Big1), GasLimit: core.CalcGasLimit(parent, w.gasFloor, w.gasCeil), Extra: w.extra, - Time: big.NewInt(tstamp), + Time: big.NewInt(timestamp), } // Only set the coinbase if our consensus engine is running (avoid spurious block rewards) if w.isRunning() { From 89451f7c382ad2185987ee369f16416f89c28a7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 29 Aug 2018 18:24:32 +0300 Subject: [PATCH 69/96] params, swarm: release geth v1.8.15 and swarm 0.3.3 --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index f65236a85c..12f4111507 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 15 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 15 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 5672e19f81..4cd095fb7c 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 3 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 3 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 62e94895da80838ec694b4ba19eeac80c9dc88c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 29 Aug 2018 18:27:43 +0300 Subject: [PATCH 70/96] params, swarm: begin geth v1.8.16 and swarm v0.3.4 cycle --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 12f4111507..a178af8f5f 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 15 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 16 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 4cd095fb7c..a13ca64afb 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 3 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 4 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 5c0954afffe223ea61ea42dbc74ec7dbdc3663a1 Mon Sep 17 00:00:00 2001 From: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com> Date: Mon, 3 Sep 2018 22:47:20 +0800 Subject: [PATCH 71/96] p2p/discv5: make idx bounds checking more sound (#17571) --- p2p/discv5/net.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index b93c93d648..a6cabf0803 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -1228,7 +1228,7 @@ func (net *Network) checkTopicRegister(data *topicRegister) (*pong, error) { if rlpHash(data.Topics) != pongpkt.data.(*pong).TopicHash { return nil, errors.New("topic hash mismatch") } - if int(data.Idx) < 0 || int(data.Idx) >= len(data.Topics) { + if data.Idx >= uint(len(data.Topics)) { return nil, errors.New("topic index out of range") } return pongpkt.data.(*pong), nil From 992b77992f5da7c0301c1b4f8203e8f0d3dfb8f4 Mon Sep 17 00:00:00 2001 From: ult-bobonovski Date: Mon, 3 Sep 2018 22:49:00 +0800 Subject: [PATCH 72/96] consensus: fix comment typo (#17562) --- consensus/consensus.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/consensus.go b/consensus/consensus.go index 12ede7ff46..487b07be77 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -90,7 +90,7 @@ type Engine interface { // the result into the given channel. // // Note, the method returns immediately and will send the result async. More - // than one result may also be returned depending on the consensus algorothm. + // than one result may also be returned depending on the consensus algorithm. Seal(chain ChainReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error // SealHash returns the hash of a block prior to it being sealed. From e1c64a7d89a7cc6ae8867ee030d3c8ead677c272 Mon Sep 17 00:00:00 2001 From: Mymskmkt <1847234666@qq.com> Date: Mon, 3 Sep 2018 22:52:32 +0800 Subject: [PATCH 73/96] params: fix typo (#17552) --- params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/config.go b/params/config.go index 70a1edead4..629720550a 100644 --- a/params/config.go +++ b/params/config.go @@ -327,7 +327,7 @@ func (err *ConfigCompatError) Error() string { return fmt.Sprintf("mismatching %s in database (have %d, want %d, rewindto %d)", err.What, err.StoredConfig, err.NewConfig, err.RewindTo) } -// Rules wraps ChainConfig and is merely syntatic sugar or can be used for functions +// Rules wraps ChainConfig and is merely syntactic sugar or can be used for functions // that do not have or require information about the block. // // Rules is a one time interface meaning that it shouldn't be used in between transition From c9a0b36a5f89797672facdb2983600e22ecf6417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=ADsli=20Kristj=C3=A1nsson?= Date: Mon, 3 Sep 2018 14:56:30 +0000 Subject: [PATCH 74/96] rpc: reset client write deadline after write (#17549) This fixes an issue with websocket ping frame handling. --- rpc/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/rpc/client.go b/rpc/client.go index a2ef2ed6b6..d96189a2d8 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -487,6 +487,7 @@ func (c *Client) write(ctx context.Context, msg interface{}) error { } c.writeConn.SetWriteDeadline(deadline) err := json.NewEncoder(c.writeConn).Encode(msg) + c.writeConn.SetWriteDeadline(time.Time{}) if err != nil { c.writeConn = nil } From cc2b39bbd18a8f67ac9ceca9ee8eabfc72d0b14b Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 3 Sep 2018 16:59:23 +0200 Subject: [PATCH 75/96] consensus/ethash: increase timeout in test (#17526) This is an attempt to fix the flaky consensus/ethash tests under macOS. --- consensus/ethash/sealer_test.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go index 31d18b67c7..436359af7c 100644 --- a/consensus/ethash/sealer_test.go +++ b/consensus/ethash/sealer_test.go @@ -40,6 +40,18 @@ func TestRemoteNotify(t *testing.T) { go server.Serve(listener) + // Wait for server to start listening + var tries int + for tries = 0; tries < 10; tries++ { + conn, _ := net.DialTimeout("tcp", listener.Addr().String(), 1*time.Second) + if conn != nil { + break + } + } + if tries == 10 { + t.Fatal("tcp listener not ready for more than 10 seconds") + } + // Create the custom ethash engine ethash := NewTester([]string{"http://" + listener.Addr().String()}, false) defer ethash.Close() @@ -61,7 +73,7 @@ func TestRemoteNotify(t *testing.T) { if want := common.BytesToHash(target.Bytes()).Hex(); work[2] != want { t.Errorf("work packet target mismatch: have %s, want %s", work[2], want) } - case <-time.After(time.Second): + case <-time.After(3 * time.Second): t.Fatalf("notification timed out") } } @@ -108,7 +120,7 @@ func TestRemoteMultiNotify(t *testing.T) { for i := 0; i < cap(sink); i++ { select { case <-sink: - case <-time.After(250 * time.Millisecond): + case <-time.After(3 * time.Second): t.Fatalf("notification %d timed out", i) } } From 6fc84946203929143c51010e0a10d9fa61fb9b92 Mon Sep 17 00:00:00 2001 From: Eugene Valeyev Date: Mon, 3 Sep 2018 18:30:47 +0300 Subject: [PATCH 76/96] mobile: add whisper client (#15922) --- mobile/ethclient.go | 8 +- mobile/shhclient.go | 195 ++++++++++++++++++++++++++++++++++++++++++++ mobile/types.go | 93 +++++++++++++++++++++ 3 files changed, 294 insertions(+), 2 deletions(-) create mode 100644 mobile/shhclient.go diff --git a/mobile/ethclient.go b/mobile/ethclient.go index 66399c6b56..662125c4ad 100644 --- a/mobile/ethclient.go +++ b/mobile/ethclient.go @@ -138,7 +138,9 @@ func (ec *EthereumClient) SubscribeNewHead(ctx *Context, handler NewHeadHandler, handler.OnNewHead(&Header{header}) case err := <-rawSub.Err(): - handler.OnError(err.Error()) + if err != nil { + handler.OnError(err.Error()) + } return } } @@ -227,7 +229,9 @@ func (ec *EthereumClient) SubscribeFilterLogs(ctx *Context, query *FilterQuery, handler.OnFilterLogs(&Log{&log}) case err := <-rawSub.Err(): - handler.OnError(err.Error()) + if err != nil { + handler.OnError(err.Error()) + } return } } diff --git a/mobile/shhclient.go b/mobile/shhclient.go new file mode 100644 index 0000000000..a069c9bd40 --- /dev/null +++ b/mobile/shhclient.go @@ -0,0 +1,195 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains a wrapper for the Whisper client. + +package geth + +import ( + "github.com/ethereum/go-ethereum/whisper/shhclient" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" +) + +// WhisperClient provides access to the Ethereum APIs. +type WhisperClient struct { + client *shhclient.Client +} + +// NewWhisperClient connects a client to the given URL. +func NewWhisperClient(rawurl string) (client *WhisperClient, _ error) { + rawClient, err := shhclient.Dial(rawurl) + return &WhisperClient{rawClient}, err +} + +// GetVersion returns the Whisper sub-protocol version. +func (wc *WhisperClient) GetVersion(ctx *Context) (version string, _ error) { + return wc.client.Version(ctx.context) +} + +// Info returns diagnostic information about the whisper node. +func (wc *WhisperClient) GetInfo(ctx *Context) (info *Info, _ error) { + rawInfo, err := wc.client.Info(ctx.context) + return &Info{&rawInfo}, err +} + +// SetMaxMessageSize sets the maximal message size allowed by this node. Incoming +// and outgoing messages with a larger size will be rejected. Whisper message size +// can never exceed the limit imposed by the underlying P2P protocol (10 Mb). +func (wc *WhisperClient) SetMaxMessageSize(ctx *Context, size int32) error { + return wc.client.SetMaxMessageSize(ctx.context, uint32(size)) +} + +// SetMinimumPoW (experimental) sets the minimal PoW required by this node. +// This experimental function was introduced for the future dynamic adjustment of +// PoW requirement. If the node is overwhelmed with messages, it should raise the +// PoW requirement and notify the peers. The new value should be set relative to +// the old value (e.g. double). The old value could be obtained via shh_info call. +func (wc *WhisperClient) SetMinimumPoW(ctx *Context, pow float64) error { + return wc.client.SetMinimumPoW(ctx.context, pow) +} + +// Marks specific peer trusted, which will allow it to send historic (expired) messages. +// Note This function is not adding new nodes, the node needs to exists as a peer. +func (wc *WhisperClient) MarkTrustedPeer(ctx *Context, enode string) error { + return wc.client.MarkTrustedPeer(ctx.context, enode) +} + +// NewKeyPair generates a new public and private key pair for message decryption and encryption. +// It returns an identifier that can be used to refer to the key. +func (wc *WhisperClient) NewKeyPair(ctx *Context) (string, error) { + return wc.client.NewKeyPair(ctx.context) +} + +// AddPrivateKey stored the key pair, and returns its ID. +func (wc *WhisperClient) AddPrivateKey(ctx *Context, key []byte) (string, error) { + return wc.client.AddPrivateKey(ctx.context, key) +} + +// DeleteKeyPair delete the specifies key. +func (wc *WhisperClient) DeleteKeyPair(ctx *Context, id string) (string, error) { + return wc.client.DeleteKeyPair(ctx.context, id) +} + +// HasKeyPair returns an indication if the node has a private key or +// key pair matching the given ID. +func (wc *WhisperClient) HasKeyPair(ctx *Context, id string) (bool, error) { + return wc.client.HasKeyPair(ctx.context, id) +} + +// GetPublicKey return the public key for a key ID. +func (wc *WhisperClient) GetPublicKey(ctx *Context, id string) ([]byte, error) { + return wc.client.PublicKey(ctx.context, id) +} + +// GetPrivateKey return the private key for a key ID. +func (wc *WhisperClient) GetPrivateKey(ctx *Context, id string) ([]byte, error) { + return wc.client.PrivateKey(ctx.context, id) +} + +// NewSymmetricKey generates a random symmetric key and returns its identifier. +// Can be used encrypting and decrypting messages where the key is known to both parties. +func (wc *WhisperClient) NewSymmetricKey(ctx *Context) (string, error) { + return wc.client.NewSymmetricKey(ctx.context) +} + +// AddSymmetricKey stores the key, and returns its identifier. +func (wc *WhisperClient) AddSymmetricKey(ctx *Context, key []byte) (string, error) { + return wc.client.AddSymmetricKey(ctx.context, key) +} + +// GenerateSymmetricKeyFromPassword generates the key from password, stores it, and returns its identifier. +func (wc *WhisperClient) GenerateSymmetricKeyFromPassword(ctx *Context, passwd string) (string, error) { + return wc.client.GenerateSymmetricKeyFromPassword(ctx.context, passwd) +} + +// HasSymmetricKey returns an indication if the key associated with the given id is stored in the node. +func (wc *WhisperClient) HasSymmetricKey(ctx *Context, id string) (bool, error) { + return wc.client.HasSymmetricKey(ctx.context, id) +} + +// GetSymmetricKey returns the symmetric key associated with the given identifier. +func (wc *WhisperClient) GetSymmetricKey(ctx *Context, id string) ([]byte, error) { + return wc.client.GetSymmetricKey(ctx.context, id) +} + +// DeleteSymmetricKey deletes the symmetric key associated with the given identifier. +func (wc *WhisperClient) DeleteSymmetricKey(ctx *Context, id string) error { + return wc.client.DeleteSymmetricKey(ctx.context, id) +} + +// Post a message onto the network. +func (wc *WhisperClient) Post(ctx *Context, message *NewMessage) (string, error) { + return wc.client.Post(ctx.context, *message.newMessage) +} + +// NewHeadHandler is a client-side subscription callback to invoke on events and +// subscription failure. +type NewMessageHandler interface { + OnNewMessage(message *Message) + OnError(failure string) +} + +// SubscribeMessages subscribes to messages that match the given criteria. This method +// is only supported on bi-directional connections such as websockets and IPC. +// NewMessageFilter uses polling and is supported over HTTP. +func (wc *WhisperClient) SubscribeMessages(ctx *Context, criteria *Criteria, handler NewMessageHandler, buffer int) (*Subscription, error) { + // Subscribe to the event internally + ch := make(chan *whisper.Message, buffer) + rawSub, err := wc.client.SubscribeMessages(ctx.context, *criteria.criteria, ch) + if err != nil { + return nil, err + } + // Start up a dispatcher to feed into the callback + go func() { + for { + select { + case message := <-ch: + handler.OnNewMessage(&Message{message}) + + case err := <-rawSub.Err(): + if err != nil { + handler.OnError(err.Error()) + } + return + } + } + }() + return &Subscription{rawSub}, nil +} + +// NewMessageFilter creates a filter within the node. This filter can be used to poll +// for new messages (see FilterMessages) that satisfy the given criteria. A filter can +// timeout when it was polled for in whisper.filterTimeout. +func (wc *WhisperClient) NewMessageFilter(ctx *Context, criteria *Criteria) (string, error) { + return wc.client.NewMessageFilter(ctx.context, *criteria.criteria) +} + +// DeleteMessageFilter removes the filter associated with the given id. +func (wc *WhisperClient) DeleteMessageFilter(ctx *Context, id string) error { + return wc.client.DeleteMessageFilter(ctx.context, id) +} + +// GetFilterMessages retrieves all messages that are received between the last call to +// this function and match the criteria that where given when the filter was created. +func (wc *WhisperClient) GetFilterMessages(ctx *Context, id string) (*Messages, error) { + rawFilterMessages, err := wc.client.FilterMessages(ctx.context, id) + if err != nil { + return nil, err + } + res := make([]*whisper.Message, len(rawFilterMessages)) + copy(res, rawFilterMessages) + return &Messages{res}, nil +} diff --git a/mobile/types.go b/mobile/types.go index b2780f3076..443d07ea93 100644 --- a/mobile/types.go +++ b/mobile/types.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) // A Nonce is a 64-bit hash which proves (combined with the mix-hash) that @@ -334,3 +335,95 @@ func (r *Receipt) GetLogs() *Logs { return &Logs{r.receipt.Logs} } func (r *Receipt) GetTxHash() *Hash { return &Hash{r.receipt.TxHash} } func (r *Receipt) GetContractAddress() *Address { return &Address{r.receipt.ContractAddress} } func (r *Receipt) GetGasUsed() int64 { return int64(r.receipt.GasUsed) } + +// Info represents a diagnostic information about the whisper node. +type Info struct { + info *whisper.Info +} + +// NewMessage represents a new whisper message that is posted through the RPC. +type NewMessage struct { + newMessage *whisper.NewMessage +} + +func NewNewMessage() *NewMessage { + nm := &NewMessage{ + newMessage: new(whisper.NewMessage), + } + return nm +} + +func (nm *NewMessage) GetSymKeyID() string { return nm.newMessage.SymKeyID } +func (nm *NewMessage) SetSymKeyID(symKeyID string) { nm.newMessage.SymKeyID = symKeyID } +func (nm *NewMessage) GetPublicKey() []byte { return nm.newMessage.PublicKey } +func (nm *NewMessage) SetPublicKey(publicKey []byte) { + nm.newMessage.PublicKey = common.CopyBytes(publicKey) +} +func (nm *NewMessage) GetSig() string { return nm.newMessage.Sig } +func (nm *NewMessage) SetSig(sig string) { nm.newMessage.Sig = sig } +func (nm *NewMessage) GetTTL() int64 { return int64(nm.newMessage.TTL) } +func (nm *NewMessage) SetTTL(ttl int64) { nm.newMessage.TTL = uint32(ttl) } +func (nm *NewMessage) GetPayload() []byte { return nm.newMessage.Payload } +func (nm *NewMessage) SetPayload(payload []byte) { nm.newMessage.Payload = common.CopyBytes(payload) } +func (nm *NewMessage) GetPowTime() int64 { return int64(nm.newMessage.PowTime) } +func (nm *NewMessage) SetPowTime(powTime int64) { nm.newMessage.PowTime = uint32(powTime) } +func (nm *NewMessage) GetPowTarget() float64 { return nm.newMessage.PowTarget } +func (nm *NewMessage) SetPowTarget(powTarget float64) { nm.newMessage.PowTarget = powTarget } +func (nm *NewMessage) GetTargetPeer() string { return nm.newMessage.TargetPeer } +func (nm *NewMessage) SetTargetPeer(targetPeer string) { nm.newMessage.TargetPeer = targetPeer } +func (nm *NewMessage) GetTopic() []byte { return nm.newMessage.Topic[:] } +func (nm *NewMessage) SetTopic(topic []byte) { nm.newMessage.Topic = whisper.BytesToTopic(topic) } + +// Message represents a whisper message. +type Message struct { + message *whisper.Message +} + +func (m *Message) GetSig() []byte { return m.message.Sig } +func (m *Message) GetTTL() int64 { return int64(m.message.TTL) } +func (m *Message) GetTimestamp() int64 { return int64(m.message.Timestamp) } +func (m *Message) GetPayload() []byte { return m.message.Payload } +func (m *Message) GetPoW() float64 { return m.message.PoW } +func (m *Message) GetHash() []byte { return m.message.Hash } +func (m *Message) GetDst() []byte { return m.message.Dst } + +// Messages represents an array of messages. +type Messages struct { + messages []*whisper.Message +} + +// Size returns the number of messages in the slice. +func (m *Messages) Size() int { + return len(m.messages) +} + +// Get returns the message at the given index from the slice. +func (m *Messages) Get(index int) (message *Message, _ error) { + if index < 0 || index >= len(m.messages) { + return nil, errors.New("index out of bounds") + } + return &Message{m.messages[index]}, nil +} + +// Criteria holds various filter options for inbound messages. +type Criteria struct { + criteria *whisper.Criteria +} + +func NewCriteria(topic []byte) *Criteria { + c := &Criteria{ + criteria: new(whisper.Criteria), + } + encodedTopic := whisper.BytesToTopic(topic) + c.criteria.Topics = []whisper.TopicType{encodedTopic} + return c +} + +func (c *Criteria) GetSymKeyID() string { return c.criteria.SymKeyID } +func (c *Criteria) SetSymKeyID(symKeyID string) { c.criteria.SymKeyID = symKeyID } +func (c *Criteria) GetPrivateKeyID() string { return c.criteria.PrivateKeyID } +func (c *Criteria) SetPrivateKeyID(privateKeyID string) { c.criteria.PrivateKeyID = privateKeyID } +func (c *Criteria) GetSig() []byte { return c.criteria.Sig } +func (c *Criteria) SetSig(sig []byte) { c.criteria.Sig = common.CopyBytes(sig) } +func (c *Criteria) GetMinPow() float64 { return c.criteria.MinPow } +func (c *Criteria) SetMinPow(pow float64) { c.criteria.MinPow = pow } From 6a33954731658667056466bf7573ed1c397f4750 Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Mon, 3 Sep 2018 23:33:21 +0800 Subject: [PATCH 77/96] core, eth, trie: use common/prque (#17508) --- core/blockchain.go | 6 +- core/tx_pool.go | 6 +- eth/downloader/queue.go | 34 +++--- eth/fetcher/fetcher.go | 8 +- trie/sync.go | 6 +- vendor/gopkg.in/karalabe/cookiejar.v2/LICENSE | 25 ---- .../gopkg.in/karalabe/cookiejar.v2/README.md | 109 ------------------ .../cookiejar.v2/collections/prque/prque.go | 66 ----------- .../cookiejar.v2/collections/prque/sstack.go | 91 --------------- vendor/vendor.json | 6 - 10 files changed, 30 insertions(+), 327 deletions(-) delete mode 100755 vendor/gopkg.in/karalabe/cookiejar.v2/LICENSE delete mode 100755 vendor/gopkg.in/karalabe/cookiejar.v2/README.md delete mode 100755 vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/prque.go delete mode 100755 vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/sstack.go diff --git a/core/blockchain.go b/core/blockchain.go index 0461da7fd9..63f60ca280 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -43,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/hashicorp/golang-lru" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) var ( @@ -151,7 +151,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par chainConfig: chainConfig, cacheConfig: cacheConfig, db: db, - triegc: prque.New(), + triegc: prque.New(nil), stateCache: state.NewDatabase(db), quit: make(chan struct{}), bodyCache: bodyCache, @@ -915,7 +915,7 @@ func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types. } else { // Full but not archive node, do proper garbage collection triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive - bc.triegc.Push(root, -float32(block.NumberU64())) + bc.triegc.Push(root, -int64(block.NumberU64())) if current := block.NumberU64(); current > triesInMemory { // If we exceeded our memory allowance, flush matured singleton nodes to disk diff --git a/core/tx_pool.go b/core/tx_pool.go index 46ae2759bc..a0a6ff8510 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -26,13 +26,13 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) const ( @@ -987,11 +987,11 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) { if pending > pool.config.GlobalSlots { pendingBeforeCap := pending // Assemble a spam order to penalize large transactors first - spammers := prque.New() + spammers := prque.New(nil) for addr, list := range pool.pending { // Only evict transactions from high rollers if !pool.locals.contains(addr) && uint64(list.Len()) > pool.config.AccountSlots { - spammers.Push(addr, float32(list.Len())) + spammers.Push(addr, int64(list.Len())) } } // Gradually drop transactions from offenders diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index 8529535ba7..a0e8a6d48c 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -26,10 +26,10 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) var ( @@ -105,11 +105,11 @@ func newQueue() *queue { headerPendPool: make(map[string]*fetchRequest), headerContCh: make(chan bool), blockTaskPool: make(map[common.Hash]*types.Header), - blockTaskQueue: prque.New(), + blockTaskQueue: prque.New(nil), blockPendPool: make(map[string]*fetchRequest), blockDonePool: make(map[common.Hash]struct{}), receiptTaskPool: make(map[common.Hash]*types.Header), - receiptTaskQueue: prque.New(), + receiptTaskQueue: prque.New(nil), receiptPendPool: make(map[string]*fetchRequest), receiptDonePool: make(map[common.Hash]struct{}), resultCache: make([]*fetchResult, blockCacheItems), @@ -277,7 +277,7 @@ func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) { } // Schedule all the header retrieval tasks for the skeleton assembly q.headerTaskPool = make(map[uint64]*types.Header) - q.headerTaskQueue = prque.New() + q.headerTaskQueue = prque.New(nil) q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch) q.headerProced = 0 @@ -288,7 +288,7 @@ func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) { index := from + uint64(i*MaxHeaderFetch) q.headerTaskPool[index] = header - q.headerTaskQueue.Push(index, -float32(index)) + q.headerTaskQueue.Push(index, -int64(index)) } } @@ -334,11 +334,11 @@ func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header { } // Queue the header for content retrieval q.blockTaskPool[hash] = header - q.blockTaskQueue.Push(header, -float32(header.Number.Uint64())) + q.blockTaskQueue.Push(header, -int64(header.Number.Uint64())) if q.mode == FastSync { q.receiptTaskPool[hash] = header - q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64())) + q.receiptTaskQueue.Push(header, -int64(header.Number.Uint64())) } inserts = append(inserts, header) q.headerHead = hash @@ -436,7 +436,7 @@ func (q *queue) ReserveHeaders(p *peerConnection, count int) *fetchRequest { } // Merge all the skipped batches back for _, from := range skip { - q.headerTaskQueue.Push(from, -float32(from)) + q.headerTaskQueue.Push(from, -int64(from)) } // Assemble and return the block download request if send == 0 { @@ -542,7 +542,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common } // Merge all the skipped headers back for _, header := range skip { - taskQueue.Push(header, -float32(header.Number.Uint64())) + taskQueue.Push(header, -int64(header.Number.Uint64())) } if progress { // Wake WaitResults, resultCache was modified @@ -585,10 +585,10 @@ func (q *queue) cancel(request *fetchRequest, taskQueue *prque.Prque, pendPool m defer q.lock.Unlock() if request.From > 0 { - taskQueue.Push(request.From, -float32(request.From)) + taskQueue.Push(request.From, -int64(request.From)) } for _, header := range request.Headers { - taskQueue.Push(header, -float32(header.Number.Uint64())) + taskQueue.Push(header, -int64(header.Number.Uint64())) } delete(pendPool, request.Peer.id) } @@ -602,13 +602,13 @@ func (q *queue) Revoke(peerID string) { if request, ok := q.blockPendPool[peerID]; ok { for _, header := range request.Headers { - q.blockTaskQueue.Push(header, -float32(header.Number.Uint64())) + q.blockTaskQueue.Push(header, -int64(header.Number.Uint64())) } delete(q.blockPendPool, peerID) } if request, ok := q.receiptPendPool[peerID]; ok { for _, header := range request.Headers { - q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64())) + q.receiptTaskQueue.Push(header, -int64(header.Number.Uint64())) } delete(q.receiptPendPool, peerID) } @@ -657,10 +657,10 @@ func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, // Return any non satisfied requests to the pool if request.From > 0 { - taskQueue.Push(request.From, -float32(request.From)) + taskQueue.Push(request.From, -int64(request.From)) } for _, header := range request.Headers { - taskQueue.Push(header, -float32(header.Number.Uint64())) + taskQueue.Push(header, -int64(header.Number.Uint64())) } // Add the peer to the expiry report along the number of failed requests expiries[id] = len(request.Headers) @@ -731,7 +731,7 @@ func (q *queue) DeliverHeaders(id string, headers []*types.Header, headerProcCh } miss[request.From] = struct{}{} - q.headerTaskQueue.Push(request.From, -float32(request.From)) + q.headerTaskQueue.Push(request.From, -int64(request.From)) return 0, errors.New("delivery not accepted") } // Clean up a successful fetch and try to deliver any sub-results @@ -854,7 +854,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQ // Return all failed or missing fetches to the queue for _, header := range request.Headers { if header != nil { - taskQueue.Push(header, -float32(header.Number.Uint64())) + taskQueue.Push(header, -int64(header.Number.Uint64())) } } // Wake up WaitResults diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index 277f14b81a..f0b5e8064b 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -23,10 +23,10 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) const ( @@ -160,7 +160,7 @@ func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBloc fetching: make(map[common.Hash]*announce), fetched: make(map[common.Hash][]*announce), completing: make(map[common.Hash]*announce), - queue: prque.New(), + queue: prque.New(nil), queues: make(map[string]int), queued: make(map[common.Hash]*inject), getBlock: getBlock, @@ -299,7 +299,7 @@ func (f *Fetcher) loop() { // If too high up the chain or phase, continue later number := op.block.NumberU64() if number > height+1 { - f.queue.Push(op, -float32(number)) + f.queue.Push(op, -int64(number)) if f.queueChangeHook != nil { f.queueChangeHook(hash, true) } @@ -624,7 +624,7 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) { } f.queues[peer] = count f.queued[hash] = op - f.queue.Push(op, -float32(block.NumberU64())) + f.queue.Push(op, -int64(block.NumberU64())) if f.queueChangeHook != nil { f.queueChangeHook(op.block.Hash(), true) } diff --git a/trie/sync.go b/trie/sync.go index 88d6eb7799..67dff5a8b6 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -21,8 +21,8 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/ethdb" - "gopkg.in/karalabe/cookiejar.v2/collections/prque" ) // ErrNotRequested is returned by the trie sync when it's requested to process a @@ -84,7 +84,7 @@ func NewSync(root common.Hash, database DatabaseReader, callback LeafCallback) * database: database, membatch: newSyncMemBatch(), requests: make(map[common.Hash]*request), - queue: prque.New(), + queue: prque.New(nil), } ts.AddSubTrie(root, 0, common.Hash{}, callback) return ts @@ -242,7 +242,7 @@ func (s *Sync) schedule(req *request) { return } // Schedule the request for future retrieval - s.queue.Push(req.hash, float32(req.depth)) + s.queue.Push(req.hash, int64(req.depth)) s.requests[req.hash] = req } diff --git a/vendor/gopkg.in/karalabe/cookiejar.v2/LICENSE b/vendor/gopkg.in/karalabe/cookiejar.v2/LICENSE deleted file mode 100755 index 467d60878d..0000000000 --- a/vendor/gopkg.in/karalabe/cookiejar.v2/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -Copyright (c) 2014 Péter Szilágyi. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Alternatively, the CookieJar toolbox may be used in accordance with the terms -and conditions contained in a signed written agreement between you and the -author(s). diff --git a/vendor/gopkg.in/karalabe/cookiejar.v2/README.md b/vendor/gopkg.in/karalabe/cookiejar.v2/README.md deleted file mode 100755 index 44c420b60e..0000000000 --- a/vendor/gopkg.in/karalabe/cookiejar.v2/README.md +++ /dev/null @@ -1,109 +0,0 @@ - CookieJar - A contestant's toolbox -====================================== - -CookieJar is a small collection of common algorithms, data structures and library extensions that were deemed handy for computing competitions at one point or another. - -This toolbox is a work in progress for the time being. It may be lacking, and it may change drastically between commits (although every effort is made not to). You're welcome to use it, but it's your head on the line :) - - Installation ----------------- - -To get the package, execute: - - go get gopkg.in/karalabe/cookiejar.v2 - -To import this package, add the following line to your code: - - import "gopkg.in/karalabe/cookiejar.v2" - -For more details, see the [package documentation](http://godoc.org/gopkg.in/karalabe/cookiejar.v2). - - Contents ------------- - -Algorithms: - - Graph - - [Breadth First Search](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/graph/bfs) - - [Depth First Search](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/graph/dfs) - -Data structures: - - [Bag](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/bag) - - [Deque](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/deque) - - [Graph](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/graph) - - [Priority Queue](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/prque) - - [Queue](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/queue) - - [Set](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/set) - - [Stack](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/collections/stack) - -Extensions: - - [fmt](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/exts/fmtext) - - `Scan` and `Fscan` for `int`, `float64`, `string` and lines - - [math](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/exts/mathext) - - `Abs` for `int` - - `Min` and `Max` for `int`, `big.Int` and `big.Rat` - - `Sign` for `int` and `float64` - - [os](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/exts/osext) - - `Open` and `Create` without error codes - - [sort](http://godoc.org/gopkg.in/karalabe/cookiejar.v2/exts/sortext) - - `Sort` and `Search` for `big.Int` and `big.Rat` - - `Unique` for any `sort.Interface` - -Below are the performance results for the data structures and the complexity analysis for the algorithms. - - Performance ---------------- - -Intel(R) Core(TM) i7-2600 CPU @ 3.40GHz: -``` -- bag - - BenchmarkInsert 309 ns/op - - BenchmarkRemove 197 ns/op - - BenchmarkDo 28.1 ns/op -- deque - - BenchmarkPush 25.4 ns/op - - BenchmarkPop 6.72 ns/op -- prque - - BenchmarkPush 171 ns/op - - BenchmarkPop 947 ns/op -- queue - - BenchmarkPush 23.0 ns/op - - BenchmarkPop 5.92 ns/op -- set - - BenchmarkInsert 259 ns/op - - BenchmarkRemove 115 ns/op - - BenchmarkDo 20.9 ns/op -- stack - - BenchmarkPush 16.4 ns/op - - BenchmarkPop 6.45 ns/op -``` - - Complexity --------------- - -| Algorithm | Time complexity | Space complexity | -|:---------:|:---------------:|:----------------:| -| graph/bfs | O(E) | O(V) | -| graph/dfs | O(E) | O(E) | - - Here be dragons :) ----------------------- - -``` - . _///_, - . / ` ' '> - ) o' __/_'> - ( / _/ )_\'> - ' "__/ /_/\_> - ____/_/_/_/ - /,---, _/ / - "" /_/_/_/ - /_(_(_(_ \ - ( \_\_\\_ )\ - \'__\_\_\_\__ ).\ - //____|___\__) )_/ - | _ \'___'_( /' - \_ (-'\'___'_\ __,'_' - __) \ \\___(_ __/.__,' - ,((,-,__\ '", __\_/. __,' - '"./_._._-' -``` diff --git a/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/prque.go b/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/prque.go deleted file mode 100755 index 5c1967c658..0000000000 --- a/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/prque.go +++ /dev/null @@ -1,66 +0,0 @@ -// CookieJar - A contestant's algorithm toolbox -// Copyright (c) 2013 Peter Szilagyi. All rights reserved. -// -// CookieJar is dual licensed: use of this source code is governed by a BSD -// license that can be found in the LICENSE file. Alternatively, the CookieJar -// toolbox may be used in accordance with the terms and conditions contained -// in a signed written agreement between you and the author(s). - -// Package prque implements a priority queue data structure supporting arbitrary -// value types and float priorities. -// -// The reasoning behind using floats for the priorities vs. ints or interfaces -// was larger flexibility without sacrificing too much performance or code -// complexity. -// -// If you would like to use a min-priority queue, simply negate the priorities. -// -// Internally the queue is based on the standard heap package working on a -// sortable version of the block based stack. -package prque - -import ( - "container/heap" -) - -// Priority queue data structure. -type Prque struct { - cont *sstack -} - -// Creates a new priority queue. -func New() *Prque { - return &Prque{newSstack()} -} - -// Pushes a value with a given priority into the queue, expanding if necessary. -func (p *Prque) Push(data interface{}, priority float32) { - heap.Push(p.cont, &item{data, priority}) -} - -// Pops the value with the greates priority off the stack and returns it. -// Currently no shrinking is done. -func (p *Prque) Pop() (interface{}, float32) { - item := heap.Pop(p.cont).(*item) - return item.value, item.priority -} - -// Pops only the item from the queue, dropping the associated priority value. -func (p *Prque) PopItem() interface{} { - return heap.Pop(p.cont).(*item).value -} - -// Checks whether the priority queue is empty. -func (p *Prque) Empty() bool { - return p.cont.Len() == 0 -} - -// Returns the number of element in the priority queue. -func (p *Prque) Size() int { - return p.cont.Len() -} - -// Clears the contents of the priority queue. -func (p *Prque) Reset() { - *p = *New() -} diff --git a/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/sstack.go b/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/sstack.go deleted file mode 100755 index 9f393196ef..0000000000 --- a/vendor/gopkg.in/karalabe/cookiejar.v2/collections/prque/sstack.go +++ /dev/null @@ -1,91 +0,0 @@ -// CookieJar - A contestant's algorithm toolbox -// Copyright (c) 2013 Peter Szilagyi. All rights reserved. -// -// CookieJar is dual licensed: use of this source code is governed by a BSD -// license that can be found in the LICENSE file. Alternatively, the CookieJar -// toolbox may be used in accordance with the terms and conditions contained -// in a signed written agreement between you and the author(s). - -package prque - -// The size of a block of data -const blockSize = 4096 - -// A prioritized item in the sorted stack. -type item struct { - value interface{} - priority float32 -} - -// Internal sortable stack data structure. Implements the Push and Pop ops for -// the stack (heap) functionality and the Len, Less and Swap methods for the -// sortability requirements of the heaps. -type sstack struct { - size int - capacity int - offset int - - blocks [][]*item - active []*item -} - -// Creates a new, empty stack. -func newSstack() *sstack { - result := new(sstack) - result.active = make([]*item, blockSize) - result.blocks = [][]*item{result.active} - result.capacity = blockSize - return result -} - -// Pushes a value onto the stack, expanding it if necessary. Required by -// heap.Interface. -func (s *sstack) Push(data interface{}) { - if s.size == s.capacity { - s.active = make([]*item, blockSize) - s.blocks = append(s.blocks, s.active) - s.capacity += blockSize - s.offset = 0 - } else if s.offset == blockSize { - s.active = s.blocks[s.size/blockSize] - s.offset = 0 - } - s.active[s.offset] = data.(*item) - s.offset++ - s.size++ -} - -// Pops a value off the stack and returns it. Currently no shrinking is done. -// Required by heap.Interface. -func (s *sstack) Pop() (res interface{}) { - s.size-- - s.offset-- - if s.offset < 0 { - s.offset = blockSize - 1 - s.active = s.blocks[s.size/blockSize] - } - res, s.active[s.offset] = s.active[s.offset], nil - return -} - -// Returns the length of the stack. Required by sort.Interface. -func (s *sstack) Len() int { - return s.size -} - -// Compares the priority of two elements of the stack (higher is first). -// Required by sort.Interface. -func (s *sstack) Less(i, j int) bool { - return s.blocks[i/blockSize][i%blockSize].priority > s.blocks[j/blockSize][j%blockSize].priority -} - -// Swaps two elements in the stack. Required by sort.Interface. -func (s *sstack) Swap(i, j int) { - ib, io, jb, jo := i/blockSize, i%blockSize, j/blockSize, j%blockSize - s.blocks[ib][io], s.blocks[jb][jo] = s.blocks[jb][jo], s.blocks[ib][io] -} - -// Resets the stack, effectively clearing its contents. -func (s *sstack) Reset() { - *s = *newSstack() -} diff --git a/vendor/vendor.json b/vendor/vendor.json index fea2c804ea..6e5610460a 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -891,12 +891,6 @@ "revision": "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec", "revisionTime": "2016-12-08T18:13:25Z" }, - { - "checksumSHA1": "DQXNV0EivoHm4q+bkdahYXrjjfE=", - "path": "gopkg.in/karalabe/cookiejar.v2/collections/prque", - "revision": "8dcd6a7f4951f6ff3ee9cbb919a06d8925822e57", - "revisionTime": "2015-07-24T13:16:13Z" - }, { "checksumSHA1": "0xgs8lwcWLUffemlj+SsgKlxvDU=", "path": "gopkg.in/natefinch/npipe.v2", From 32f28a9360d26a661d55915915f12fd3c70f012b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 4 Sep 2018 10:49:18 +0200 Subject: [PATCH 78/96] core/vm, tests: update tests, enable constantinople statetests, fix SAR opcode (#17538) This commit does a few things at once: - Updates the tests to contain the latest data from ethereum/tests repo. - Enables Constantinople state tests. This is needed to be able to fuzz-test the evm with constantinople rules. - Fixes the error in opSAR that we've known about for some time. I was kind of saving it to see if we hit upon it with the random test generator, but it's difficult to both enable the tests and have the bug there -- we don't want to forget about it, so maybe it's better to just fix it. --- core/vm/instructions.go | 2 +- tests/block_test.go | 6 +++--- tests/init_test.go | 10 ++++++++++ tests/state_test.go | 3 --- tests/state_test_util.go | 13 ++++++++++++- tests/testdata | 2 +- 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index b7742e6551..ca9e775ac5 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -355,7 +355,7 @@ func opSAR(pc *uint64, interpreter *EVMInterpreter, contract *Contract, memory * defer interpreter.intPool.put(shift) // First operand back into the pool if shift.Cmp(common.Big256) >= 0 { - if value.Sign() > 0 { + if value.Sign() >= 0 { value.SetUint64(0) } else { value.SetInt64(-1) diff --git a/tests/block_test.go b/tests/block_test.go index 669d3ca08a..c911199299 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -30,11 +30,11 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`^bcForgedTest/bcForkUncle\.json`) bt.skipLoad(`^bcMultiChainTest/(ChainAtoChainB_blockorder|CallContractFromNotBestBlock)`) bt.skipLoad(`^bcTotalDifficultyTest/(lotsOfLeafs|lotsOfBranches|sideChainWithMoreTransactions)`) - // Constantinople is not implemented yet. - bt.skipLoad(`(?i)(constantinople)`) + // This test is broken + bt.fails(`blockhashNonConstArg_Constantinople`, "Broken test") // Still failing tests - bt.skipLoad(`^bcWalletTest.*_Byzantium$`) + // bt.skipLoad(`^bcWalletTest.*_Byzantium$`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { if err := bt.checkFailure(t, name, test.Run()); err != nil { diff --git a/tests/init_test.go b/tests/init_test.go index 26e919d24b..90a74448a8 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -91,6 +91,7 @@ type testMatcher struct { failpat []testFailure skiploadpat []*regexp.Regexp skipshortpat []*regexp.Regexp + whitelistpat *regexp.Regexp } type testConfig struct { @@ -121,6 +122,10 @@ func (tm *testMatcher) fails(pattern string, reason string) { tm.failpat = append(tm.failpat, testFailure{regexp.MustCompile(pattern), reason}) } +func (tm *testMatcher) whitelist(pattern string) { + tm.whitelistpat = regexp.MustCompile(pattern) +} + // config defines chain config for tests matching the pattern. func (tm *testMatcher) config(pattern string, cfg params.ChainConfig) { tm.configpat = append(tm.configpat, testConfig{regexp.MustCompile(pattern), cfg}) @@ -208,6 +213,11 @@ func (tm *testMatcher) runTestFile(t *testing.T, path, name string, runTest inte if r, _ := tm.findSkip(name); r != "" { t.Skip(r) } + if tm.whitelistpat != nil { + if !tm.whitelistpat.MatchString(name) { + t.Skip("Skipped by whitelist") + } + } t.Parallel() // Load the file as map[string]. diff --git a/tests/state_test.go b/tests/state_test.go index adec4feb2b..b61a1ca285 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -44,9 +44,6 @@ func TestState(t *testing.T) { key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) name := name + "/" + key t.Run(key, func(t *testing.T) { - if subtest.Fork == "Constantinople" { - t.Skip("constantinople not supported yet") - } withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error { _, err := test.Run(subtest, vmconfig) return st.checkFailure(t, name, err) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 84581fae18..5d2251e529 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -146,7 +146,18 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config) (*state.StateD if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { return statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } - root, _ := statedb.Commit(config.IsEIP158(block.Number())) + // Commit block + statedb.Commit(config.IsEIP158(block.Number())) + // Add 0-value mining reward. This only makes a difference in the cases + // where + // - the coinbase suicided, or + // - there are only 'bad' transactions, which aren't executed. In those cases, + // the coinbase gets no txfee, so isn't created, and thus needs to be touched + statedb.AddBalance(block.Coinbase(), new(big.Int)) + // And _now_ get the state root + root := statedb.IntermediateRoot(config.IsEIP158(block.Number())) + // N.B: We need to do this in a two-step process, because the first Commit takes care + // of suicides, and we need to touch the coinbase _after_ it has potentially suicided. if root != common.Hash(post.Root) { return statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } diff --git a/tests/testdata b/tests/testdata index 2bb0c3da3b..ad2184adca 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 2bb0c3da3bbb15c528bcef2a7e5ac4bd73f81f87 +Subproject commit ad2184adca367c0b68c65b44519dba16e1d0b9e2 From 003e031994327ab24900aa95e0f516a13e97f76f Mon Sep 17 00:00:00 2001 From: dipingxian2 <39109351+dipingxian2@users.noreply.github.com> Date: Tue, 4 Sep 2018 19:16:49 +0800 Subject: [PATCH 79/96] cmd/faucet: remove trailing newline in password (#17558) Fixes #17557 --- cmd/faucet/faucet.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 6799060275..cfe4e45f11 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -157,7 +157,8 @@ func main() { if blob, err = ioutil.ReadFile(*accPassFlag); err != nil { log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err) } - pass := string(blob) + // Delete trailing newline in password + pass := strings.TrimSuffix(string(blob), "\n") ks := keystore.NewKeyStore(filepath.Join(os.Getenv("HOME"), ".faucet", "keys"), keystore.StandardScryptN, keystore.StandardScryptP) if blob, err = ioutil.ReadFile(*accJSONFlag); err != nil { From 84084df26c29c3e35075a0624ca7f5bf204b01e8 Mon Sep 17 00:00:00 2001 From: Evangelos Pappas Date: Tue, 4 Sep 2018 12:45:27 +0100 Subject: [PATCH 80/96] cmd/ethkey: fix the README to match updated commands (#17332) --- cmd/ethkey/README.md | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/cmd/ethkey/README.md b/cmd/ethkey/README.md index cf72ba43d7..48d3c9e9b7 100644 --- a/cmd/ethkey/README.md +++ b/cmd/ethkey/README.md @@ -21,21 +21,33 @@ Private key information can be printed by using the `--private` flag; make sure to use this feature with great caution! -### `ethkey sign ` +### `ethkey signmessage ` Sign the message with a keyfile. It is possible to refer to a file containing the message. +To sign a message contained in a file, use the `--msgfile` flag. -### `ethkey verify
` +### `ethkey verifymessage
` Verify the signature of the message. It is possible to refer to a file containing the message. +To sign a message contained in a file, use the --msgfile flag. + + +### `ethkey changepassphrase ` + +Change the passphrase of a keyfile. +use the `--newpasswordfile` to point to the new password file. ## Passphrases For every command that uses a keyfile, you will be prompted to provide the passphrase for decrypting the keyfile. To avoid this message, it is possible -to pass the passphrase by using the `--passphrase` flag pointing to a file that +to pass the passphrase by using the `--passwordfile` flag pointing to a file that contains the passphrase. + +## JSON + +In case you need to output the result in a JSON format, you shall by using the `--json` flag. From 3e8184006189b57e2d998313ef0fa75429b0c5bc Mon Sep 17 00:00:00 2001 From: ligi Date: Tue, 4 Sep 2018 14:12:16 +0200 Subject: [PATCH 81/96] common: fix typo (#17582) Fixes #17581 --- common/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/types.go b/common/types.go index 71fe5c95cd..a4b9995267 100644 --- a/common/types.go +++ b/common/types.go @@ -34,7 +34,7 @@ import ( const ( // HashLength is the expected length of the hash HashLength = 32 - // AddressLength is the expected length of the adddress + // AddressLength is the expected length of the address AddressLength = 20 ) From 661aa4dc20bdd3ac7536fe8d7c41273c7433a1d8 Mon Sep 17 00:00:00 2001 From: Richard Littauer Date: Tue, 4 Sep 2018 08:13:50 -0400 Subject: [PATCH 82/96] .github: slight edits to No Response template (#17475) The template was not grammatical to me as it was. I have edited the language a tiny bit to make the close comment more clear and less abrasive. --- .github/no-response.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/no-response.yml b/.github/no-response.yml index a6227159d1..b6e96efdc6 100644 --- a/.github/no-response.yml +++ b/.github/no-response.yml @@ -7,5 +7,5 @@ closeComment: > This issue has been automatically closed because there has been no response to our request for more information from the original author. With only the information that is currently in the issue, we don't have enough information - to take action. Please reach out if you have or find the answers we need so - that we can investigate further. + to take action. Please reach out if you have more relevant information or + answers to our questions so that we can investigate further. From beee7a52e08f6e948d4cca9de8ff59d5b8305917 Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 4 Sep 2018 14:29:00 +0200 Subject: [PATCH 83/96] cmd/swarm: added scaling test for ACT manifests (#17496) --- cmd/swarm/access_test.go | 43 ++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index 7a8bcf9d3d..ed589f9f47 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -34,12 +34,15 @@ import ( "time" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/ecies" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" ) +var DefaultCurve = crypto.S256() + // TestAccessPassword tests for the correct creation of an ACT manifest protected by a password. // The test creates bogus content, uploads it encrypted, then creates the wrapping manifest with the Access entry // The parties participating - node (publisher), uploads to second node then disappears. Content which was uploaded @@ -359,11 +362,22 @@ func TestAccessPK(t *testing.T) { } } +// TestAccessACT tests the creation of the ACT manifest end-to-end, without any bogus entries (i.e. default scenario = 3 nodes 1 unauthorized) +func TestAccessACT(t *testing.T) { + testAccessACT(t, 0) +} + +// TestAccessACTScale tests the creation of the ACT manifest end-to-end, with 1000 bogus entries (i.e. 1000 EC keys + default scenario = 3 nodes 1 unauthorized = 1003 keys in the ACT manifest) +func TestAccessACTScale(t *testing.T) { + testAccessACT(t, 1000) +} + // TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data // set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through -// one of the nodes then disappears. -func TestAccessACT(t *testing.T) { +// one of the nodes then disappears. If `bogusEntries` is bigger than 0, the test will generate the number of bogus act entries +// to test what happens at scale +func testAccessACT(t *testing.T, bogusEntries int) { // Setup Swarm and upload a test file to it cluster := newTestCluster(t, 3) defer cluster.Shutdown() @@ -415,19 +429,36 @@ func TestAccessACT(t *testing.T) { grantees = append(grantees, hex.EncodeToString(granteePubKey)) } - granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list.csv") + if bogusEntries > 0 { + bogusGrantees := []string{} + + for i := 0; i < bogusEntries; i++ { + prv, err := ecies.GenerateKey(rand.Reader, DefaultCurve, nil) + if err != nil { + t.Fatal(err) + } + bogusGrantees = append(bogusGrantees, hex.EncodeToString(crypto.CompressPubkey(&prv.ExportECDSA().PublicKey))) + } + r2 := gorand.New(gorand.NewSource(time.Now().UnixNano())) + for i := 0; i < len(grantees); i++ { + insertAtIdx := r2.Intn(len(bogusGrantees)) + bogusGrantees = append(bogusGrantees[:insertAtIdx], append([]string{grantees[i]}, bogusGrantees[insertAtIdx:]...)...) + } + grantees = bogusGrantees + } + + granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list") if err != nil { t.Fatal(err) } + defer granteesPubkeyListFile.Close() + defer os.Remove(granteesPubkeyListFile.Name()) _, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n")) if err != nil { t.Fatal(err) } - defer granteesPubkeyListFile.Close() - defer os.Remove(granteesPubkeyListFile.Name()) - publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp") if err != nil { t.Fatal(err) From 42bd67bd6fd599a5583452f011152a0c26c9dff4 Mon Sep 17 00:00:00 2001 From: Diep Pham Date: Tue, 4 Sep 2018 22:53:28 +0700 Subject: [PATCH 84/96] accounts/abi: fix unpacking of negative int256 (#17583) --- accounts/abi/unpack.go | 28 +++++++++++++++++++++++++--- accounts/abi/unpack_test.go | 5 +++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index 793d515adf..d5875140cc 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -25,8 +25,17 @@ import ( "github.com/ethereum/go-ethereum/common" ) +var ( + maxUint256 = big.NewInt(0).Add( + big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil), + big.NewInt(-1)) + maxInt256 = big.NewInt(0).Add( + big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil), + big.NewInt(-1)) +) + // reads the integer based on its kind -func readInteger(kind reflect.Kind, b []byte) interface{} { +func readInteger(typ byte, kind reflect.Kind, b []byte) interface{} { switch kind { case reflect.Uint8: return b[len(b)-1] @@ -45,7 +54,20 @@ func readInteger(kind reflect.Kind, b []byte) interface{} { case reflect.Int64: return int64(binary.BigEndian.Uint64(b[len(b)-8:])) default: - return new(big.Int).SetBytes(b) + // the only case lefts for integer is int256/uint256. + // big.SetBytes can't tell if a number is negative, positive on itself. + // On EVM, if the returned number > max int256, it is negative. + ret := new(big.Int).SetBytes(b) + if typ == UintTy { + return ret + } + + if ret.Cmp(maxInt256) > 0 { + ret.Add(maxUint256, big.NewInt(0).Neg(ret)) + ret.Add(ret, big.NewInt(1)) + ret.Neg(ret) + } + return ret } } @@ -179,7 +201,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { case StringTy: // variable arrays are written at the end of the return bytes return string(output[begin : begin+end]), nil case IntTy, UintTy: - return readInteger(t.Kind, returnOutput), nil + return readInteger(t.T, t.Kind, returnOutput), nil case BoolTy: return readBool(returnOutput) case AddressTy: diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index bdbab10b4f..97552b90cf 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -117,6 +117,11 @@ var unpackTests = []unpackTest{ enc: "0000000000000000000000000000000000000000000000000000000000000001", want: big.NewInt(1), }, + { + def: `[{"type": "int256"}]`, + enc: "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + want: big.NewInt(-1), + }, { def: `[{"type": "address"}]`, enc: "0000000000000000000000000100000000000000000000000000000000000000", From cf33d8b83ce78d1e79cd8c43a21070b2050d5c7e Mon Sep 17 00:00:00 2001 From: Hyung-Kyu Hqueue Choi Date: Wed, 5 Sep 2018 17:29:51 +0900 Subject: [PATCH 85/96] core: fix typo in comment (#17586) --- core/state_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state_processor.go b/core/state_processor.go index 1a91a57ab4..503a35d16a 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -110,7 +110,7 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo *usedGas += gas // Create a new receipt for the transaction, storing the intermediate root and gas used by the tx - // based on the eip phase, we're passing wether the root touch-delete accounts. + // based on the eip phase, we're passing whether the root touch-delete accounts. receipt := types.NewReceipt(root, failed, *usedGas) receipt.TxHash = tx.Hash() receipt.GasUsed = gas From 8711e2b6366109912057e8fb20add325a1051a4e Mon Sep 17 00:00:00 2001 From: b00ris Date: Wed, 5 Sep 2018 11:57:45 +0300 Subject: [PATCH 86/96] whisper: add light mode check to handshake (#16725) --- cmd/geth/config.go | 3 ++ cmd/geth/main.go | 1 + cmd/utils/flags.go | 7 ++++ whisper/whisperv6/api.go | 8 ++--- whisper/whisperv6/config.go | 10 +++--- whisper/whisperv6/peer.go | 10 +++++- whisper/whisperv6/peer_test.go | 61 ++++++++++++++++++++++++++++++++++ whisper/whisperv6/whisper.go | 44 +++++++++++++++++++----- 8 files changed, 126 insertions(+), 18 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index e6bd4d5bef..b0749d2329 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -168,6 +168,9 @@ func makeFullNode(ctx *cli.Context) *node.Node { if ctx.GlobalIsSet(utils.WhisperMinPOWFlag.Name) { cfg.Shh.MinimumAcceptedPOW = ctx.Float64(utils.WhisperMinPOWFlag.Name) } + if ctx.GlobalIsSet(utils.WhisperRestrictConnectionBetweenLightClientsFlag.Name) { + cfg.Shh.RestrictConnectionBetweenLightClients = true + } utils.RegisterShhService(stack, &cfg.Shh) } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 4d7e98698c..134d5a4c01 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -151,6 +151,7 @@ var ( utils.WhisperEnabledFlag, utils.WhisperMaxMessageSizeFlag, utils.WhisperMinPOWFlag, + utils.WhisperRestrictConnectionBetweenLightClientsFlag, } metricsFlags = []cli.Flag{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 495bfe13e0..f431621baf 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -567,6 +567,10 @@ var ( Usage: "Minimum POW accepted", Value: whisper.DefaultMinimumPoW, } + WhisperRestrictConnectionBetweenLightClientsFlag = cli.BoolFlag{ + Name: "shh.restrict-light", + Usage: "Restrict connection between two whisper light clients", + } // Metrics flags MetricsEnabledFlag = cli.BoolFlag{ @@ -1099,6 +1103,9 @@ func SetShhConfig(ctx *cli.Context, stack *node.Node, cfg *whisper.Config) { if ctx.GlobalIsSet(WhisperMinPOWFlag.Name) { cfg.MinimumAcceptedPOW = ctx.GlobalFloat64(WhisperMinPOWFlag.Name) } + if ctx.GlobalIsSet(WhisperRestrictConnectionBetweenLightClientsFlag.Name) { + cfg.RestrictConnectionBetweenLightClients = true + } } // SetEthConfig applies eth-related command line flags to the config. diff --git a/whisper/whisperv6/api.go b/whisper/whisperv6/api.go index d729e79c35..e1dab7b226 100644 --- a/whisper/whisperv6/api.go +++ b/whisper/whisperv6/api.go @@ -195,14 +195,14 @@ func (api *PublicWhisperAPI) DeleteSymKey(ctx context.Context, id string) bool { // MakeLightClient turns the node into light client, which does not forward // any incoming messages, and sends only messages originated in this node. func (api *PublicWhisperAPI) MakeLightClient(ctx context.Context) bool { - api.w.lightClient = true - return api.w.lightClient + api.w.SetLightClientMode(true) + return api.w.LightClientMode() } // CancelLightClient cancels light client mode. func (api *PublicWhisperAPI) CancelLightClient(ctx context.Context) bool { - api.w.lightClient = false - return !api.w.lightClient + api.w.SetLightClientMode(false) + return !api.w.LightClientMode() } //go:generate gencodec -type NewMessage -field-override newMessageOverride -out gen_newmessage_json.go diff --git a/whisper/whisperv6/config.go b/whisper/whisperv6/config.go index 61419de007..38eb9551cc 100644 --- a/whisper/whisperv6/config.go +++ b/whisper/whisperv6/config.go @@ -18,12 +18,14 @@ package whisperv6 // Config represents the configuration state of a whisper node. type Config struct { - MaxMessageSize uint32 `toml:",omitempty"` - MinimumAcceptedPOW float64 `toml:",omitempty"` + MaxMessageSize uint32 `toml:",omitempty"` + MinimumAcceptedPOW float64 `toml:",omitempty"` + RestrictConnectionBetweenLightClients bool `toml:",omitempty"` } // DefaultConfig represents (shocker!) the default configuration. var DefaultConfig = Config{ - MaxMessageSize: DefaultMaxMessageSize, - MinimumAcceptedPOW: DefaultMinimumPoW, + MaxMessageSize: DefaultMaxMessageSize, + MinimumAcceptedPOW: DefaultMinimumPoW, + RestrictConnectionBetweenLightClients: true, } diff --git a/whisper/whisperv6/peer.go b/whisper/whisperv6/peer.go index 79cc212708..621d512081 100644 --- a/whisper/whisperv6/peer.go +++ b/whisper/whisperv6/peer.go @@ -79,11 +79,14 @@ func (peer *Peer) stop() { func (peer *Peer) handshake() error { // Send the handshake status message asynchronously errc := make(chan error, 1) + isLightNode := peer.host.LightClientMode() + isRestrictedLightNodeConnection := peer.host.LightClientModeConnectionRestricted() go func() { pow := peer.host.MinPow() powConverted := math.Float64bits(pow) bloom := peer.host.BloomFilter() - errc <- p2p.SendItems(peer.ws, statusCode, ProtocolVersion, powConverted, bloom) + + errc <- p2p.SendItems(peer.ws, statusCode, ProtocolVersion, powConverted, bloom, isLightNode) }() // Fetch the remote status packet and verify protocol match @@ -127,6 +130,11 @@ func (peer *Peer) handshake() error { } } + isRemotePeerLightNode, err := s.Bool() + if isRemotePeerLightNode && isLightNode && isRestrictedLightNodeConnection { + return fmt.Errorf("peer [%x] is useless: two light client communication restricted", peer.ID()) + } + if err := <-errc; err != nil { return fmt.Errorf("peer [%x] failed to send status packet: %v", peer.ID(), err) } diff --git a/whisper/whisperv6/peer_test.go b/whisper/whisperv6/peer_test.go index 0c9b380901..fe31922cb2 100644 --- a/whisper/whisperv6/peer_test.go +++ b/whisper/whisperv6/peer_test.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/nat" + "github.com/ethereum/go-ethereum/rlp" ) var keys = []string{ @@ -507,3 +508,63 @@ func waitForServersToStart(t *testing.T) { } t.Fatalf("Failed to start all the servers, running: %d", started) } + +//two generic whisper node handshake +func TestPeerHandshakeWithTwoFullNode(t *testing.T) { + w1 := Whisper{} + p1 := newPeer(&w1, p2p.NewPeer(discover.NodeID{}, "test", []p2p.Cap{}), &rwStub{[]interface{}{ProtocolVersion, uint64(123), make([]byte, BloomFilterSize), false}}) + err := p1.handshake() + if err != nil { + t.Fatal() + } +} + +//two generic whisper node handshake. one don't send light flag +func TestHandshakeWithOldVersionWithoutLightModeFlag(t *testing.T) { + w1 := Whisper{} + p1 := newPeer(&w1, p2p.NewPeer(discover.NodeID{}, "test", []p2p.Cap{}), &rwStub{[]interface{}{ProtocolVersion, uint64(123), make([]byte, BloomFilterSize)}}) + err := p1.handshake() + if err != nil { + t.Fatal() + } +} + +//two light nodes handshake. restriction disabled +func TestTwoLightPeerHandshakeRestrictionOff(t *testing.T) { + w1 := Whisper{} + w1.settings.Store(restrictConnectionBetweenLightClientsIdx, false) + w1.SetLightClientMode(true) + p1 := newPeer(&w1, p2p.NewPeer(discover.NodeID{}, "test", []p2p.Cap{}), &rwStub{[]interface{}{ProtocolVersion, uint64(123), make([]byte, BloomFilterSize), true}}) + err := p1.handshake() + if err != nil { + t.FailNow() + } +} + +//two light nodes handshake. restriction enabled +func TestTwoLightPeerHandshakeError(t *testing.T) { + w1 := Whisper{} + w1.settings.Store(restrictConnectionBetweenLightClientsIdx, true) + w1.SetLightClientMode(true) + p1 := newPeer(&w1, p2p.NewPeer(discover.NodeID{}, "test", []p2p.Cap{}), &rwStub{[]interface{}{ProtocolVersion, uint64(123), make([]byte, BloomFilterSize), true}}) + err := p1.handshake() + if err == nil { + t.FailNow() + } +} + +type rwStub struct { + payload []interface{} +} + +func (stub *rwStub) ReadMsg() (p2p.Msg, error) { + size, r, err := rlp.EncodeToReader(stub.payload) + if err != nil { + return p2p.Msg{}, err + } + return p2p.Msg{Code: statusCode, Size: uint32(size), Payload: r}, nil +} + +func (stub *rwStub) WriteMsg(m p2p.Msg) error { + return nil +} diff --git a/whisper/whisperv6/whisper.go b/whisper/whisperv6/whisper.go index be633e7ff6..eb713f84ee 100644 --- a/whisper/whisperv6/whisper.go +++ b/whisper/whisperv6/whisper.go @@ -49,12 +49,14 @@ type Statistics struct { } const ( - maxMsgSizeIdx = iota // Maximal message length allowed by the whisper node - overflowIdx // Indicator of message queue overflow - minPowIdx // Minimal PoW required by the whisper node - minPowToleranceIdx // Minimal PoW tolerated by the whisper node for a limited time - bloomFilterIdx // Bloom filter for topics of interest for this node - bloomFilterToleranceIdx // Bloom filter tolerated by the whisper node for a limited time + maxMsgSizeIdx = iota // Maximal message length allowed by the whisper node + overflowIdx // Indicator of message queue overflow + minPowIdx // Minimal PoW required by the whisper node + minPowToleranceIdx // Minimal PoW tolerated by the whisper node for a limited time + bloomFilterIdx // Bloom filter for topics of interest for this node + bloomFilterToleranceIdx // Bloom filter tolerated by the whisper node for a limited time + lightClientModeIdx // Light client mode. (does not forward any messages) + restrictConnectionBetweenLightClientsIdx // Restrict connection between two light clients ) // Whisper represents a dark communication interface through the Ethereum @@ -82,8 +84,6 @@ type Whisper struct { syncAllowance int // maximum time in seconds allowed to process the whisper-related messages - lightClient bool // indicates is this node is pure light client (does not forward any messages) - statsMu sync.Mutex // guard stats stats Statistics // Statistics of whisper node @@ -113,6 +113,7 @@ func New(cfg *Config) *Whisper { whisper.settings.Store(minPowIdx, cfg.MinimumAcceptedPOW) whisper.settings.Store(maxMsgSizeIdx, cfg.MaxMessageSize) whisper.settings.Store(overflowIdx, false) + whisper.settings.Store(restrictConnectionBetweenLightClientsIdx, cfg.RestrictConnectionBetweenLightClients) // p2p whisper sub protocol handler whisper.protocol = p2p.Protocol{ @@ -276,6 +277,31 @@ func (whisper *Whisper) SetMinimumPowTest(val float64) { whisper.settings.Store(minPowToleranceIdx, val) } +//SetLightClientMode makes node light client (does not forward any messages) +func (whisper *Whisper) SetLightClientMode(v bool) { + whisper.settings.Store(lightClientModeIdx, v) +} + +//LightClientMode indicates is this node is light client (does not forward any messages) +func (whisper *Whisper) LightClientMode() bool { + val, exist := whisper.settings.Load(lightClientModeIdx) + if !exist || val == nil { + return false + } + v, ok := val.(bool) + return v && ok +} + +//LightClientModeConnectionRestricted indicates that connection to light client in light client mode not allowed +func (whisper *Whisper) LightClientModeConnectionRestricted() bool { + val, exist := whisper.settings.Load(restrictConnectionBetweenLightClientsIdx) + if !exist || val == nil { + return false + } + v, ok := val.(bool) + return v && ok +} + func (whisper *Whisper) notifyPeersAboutPowRequirementChange(pow float64) { arr := whisper.getPeers() for _, p := range arr { @@ -672,7 +698,7 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error { trouble := false for _, env := range envelopes { - cached, err := whisper.add(env, whisper.lightClient) + cached, err := whisper.add(env, whisper.LightClientMode()) if err != nil { trouble = true log.Error("bad envelope received, peer will be disconnected", "peer", p.peer.ID(), "err", err) From 5918b88a8f8be34f39b8df94d146f77ac9b4fbfc Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 5 Sep 2018 11:33:07 +0200 Subject: [PATCH 87/96] cmd/swarm: added publisher key assertion to act tests (#17471) --- cmd/swarm/access_test.go | 44 ++++++++++++++++++++++++++++++++++++---- cmd/swarm/main.go | 24 ++++++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index ed589f9f47..384d256306 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -145,7 +145,9 @@ func TestAccessPassword(t *testing.T) { if a.KdfParams == nil { t.Fatal("manifest access kdf params is nil") } - + if a.Publisher != "" { + t.Fatal("should be empty") + } client := swarm.NewClient(cluster.Nodes[0].URL) hash, err := client.UploadManifest(&m, false) @@ -222,7 +224,7 @@ func TestAccessPassword(t *testing.T) { // the test will fail if the proxy's given private key is not granted on the ACT. func TestAccessPK(t *testing.T) { // Setup Swarm and upload a test file to it - cluster := newTestCluster(t, 1) + cluster := newTestCluster(t, 2) defer cluster.Shutdown() // create a tmp file @@ -302,6 +304,20 @@ func TestAccessPK(t *testing.T) { t.Fatalf("stdout not matched") } + //get the public key from the publisher directory + publicKeyFromDataDir := runSwarm(t, + "--bzzaccount", + publisherAccount.Address.String(), + "--password", + passFile.Name(), + "--datadir", + publisherDir, + "print-keys", + "--compressed", + ) + _, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+") + publicKeyFromDataDir.ExpectExit() + pkComp := strings.Split(publicKeyString[0], "=")[1] var m api.Manifest err = json.Unmarshal([]byte(matches[0]), &m) @@ -335,7 +351,9 @@ func TestAccessPK(t *testing.T) { if a.KdfParams != nil { t.Fatal("manifest access kdf params should be nil") } - + if a.Publisher != pkComp { + t.Fatal("publisher key did not match") + } client := swarm.NewClient(cluster.Nodes[0].URL) hash, err := client.UploadManifest(&m, false) @@ -499,6 +517,22 @@ func testAccessACT(t *testing.T, bogusEntries int) { if len(matches) == 0 { t.Fatalf("stdout not matched") } + + //get the public key from the publisher directory + publicKeyFromDataDir := runSwarm(t, + "--bzzaccount", + publisherAccount.Address.String(), + "--password", + passFile.Name(), + "--datadir", + publisherDir, + "print-keys", + "--compressed", + ) + _, publicKeyString := publicKeyFromDataDir.ExpectRegexp(".+") + publicKeyFromDataDir.ExpectExit() + pkComp := strings.Split(publicKeyString[0], "=")[1] + hash := matches[0] m, _, err := client.DownloadManifest(hash) if err != nil { @@ -531,7 +565,9 @@ func testAccessACT(t *testing.T, bogusEntries int) { if a.KdfParams != nil { t.Fatal("manifest access kdf params should be nil") } - + if a.Publisher != pkComp { + t.Fatal("publisher key did not match") + } httpClient := &http.Client{} // all nodes except the skipped node should be able to decrypt the content diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 637ae06e96..e654409371 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -18,6 +18,7 @@ package main import ( "crypto/ecdsa" + "encoding/hex" "fmt" "io/ioutil" "os" @@ -208,6 +209,10 @@ var ( Name: "data", Usage: "Initializes the resource with the given hex-encoded data. Data must be prefixed by 0x", } + SwarmCompressedFlag = cli.BoolFlag{ + Name: "compressed", + Usage: "Prints encryption keys in compressed form", + } ) //declare a few constant error messages, useful for later error check comparisons in test @@ -252,6 +257,14 @@ func init() { Usage: "Print version numbers", Description: "The output of this command is supposed to be machine-readable", }, + { + Action: keys, + CustomHelpTemplate: helpTemplate, + Name: "print-keys", + Flags: []cli.Flag{SwarmCompressedFlag}, + Usage: "Print public key information", + Description: "The output of this command is supposed to be machine-readable", + }, { Action: upload, CustomHelpTemplate: helpTemplate, @@ -580,6 +593,17 @@ func main() { } } +func keys(ctx *cli.Context) error { + privateKey := getPrivKey(ctx) + pub := hex.EncodeToString(crypto.FromECDSAPub(&privateKey.PublicKey)) + pubCompressed := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)) + if !ctx.Bool(SwarmCompressedFlag.Name) { + fmt.Println(fmt.Sprintf("publicKey=%s", pub)) + } + fmt.Println(fmt.Sprintf("publicKeyCompressed=%s", pubCompressed)) + return nil +} + func version(ctx *cli.Context) error { fmt.Println(strings.Title(clientIdentifier)) fmt.Println("Version:", sv.VersionWithMeta) From 4c15ffffdd0aaf3b834be88ff7e7d9ea938c45b4 Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 6 Sep 2018 12:08:39 +0200 Subject: [PATCH 88/96] swarm/api/http: added a regression test for resolver bug from #17483 (#17502) --- swarm/api/client/client_test.go | 14 ++-- swarm/api/http/response_test.go | 10 +-- swarm/api/http/server_test.go | 143 ++++++++++++++++++++++++++++++-- swarm/testutil/http.go | 4 +- 4 files changed, 148 insertions(+), 23 deletions(-) diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go index 2212f5c4c3..f9312d48fe 100644 --- a/swarm/api/client/client_test.go +++ b/swarm/api/client/client_test.go @@ -47,7 +47,7 @@ func TestClientUploadDownloadRawEncrypted(t *testing.T) { } func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() client := NewClient(srv.URL) @@ -88,7 +88,7 @@ func TestClientUploadDownloadFilesEncrypted(t *testing.T) { } func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() client := NewClient(srv.URL) @@ -186,7 +186,7 @@ func newTestDirectory(t *testing.T) string { // TestClientUploadDownloadDirectory tests uploading and downloading a // directory of files to a swarm manifest func TestClientUploadDownloadDirectory(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dir := newTestDirectory(t) @@ -252,7 +252,7 @@ func TestClientFileListEncrypted(t *testing.T) { } func testClientFileList(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() dir := newTestDirectory(t) @@ -310,7 +310,7 @@ func testClientFileList(toEncrypt bool, t *testing.T) { // TestClientMultipartUpload tests uploading files to swarm using a multipart // upload func TestClientMultipartUpload(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // define an uploader which uploads testDirFiles with some data @@ -376,7 +376,7 @@ func TestClientCreateResourceMultihash(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) client := NewClient(srv.URL) defer srv.Close() @@ -439,7 +439,7 @@ func TestClientCreateUpdateResource(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) client := NewClient(srv.URL) defer srv.Close() diff --git a/swarm/api/http/response_test.go b/swarm/api/http/response_test.go index 2e24bda6d1..50a704be6d 100644 --- a/swarm/api/http/response_test.go +++ b/swarm/api/http/response_test.go @@ -29,7 +29,7 @@ import ( ) func TestError(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -55,7 +55,7 @@ func TestError(t *testing.T) { } func Test404Page(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -81,7 +81,7 @@ func Test404Page(t *testing.T) { } func Test500Page(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -106,7 +106,7 @@ func Test500Page(t *testing.T) { } } func Test500PageWith0xHashPrefix(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response @@ -136,7 +136,7 @@ func Test500PageWith0xHashPrefix(t *testing.T) { } func TestJsonResponse(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() var resp *http.Response diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 7934e37eba..891deb8817 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -27,6 +27,7 @@ import ( "fmt" "io" "io/ioutil" + "math/big" "mime/multipart" "net/http" "os" @@ -36,6 +37,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" @@ -114,7 +116,7 @@ func TestBzzResourceMultihash(t *testing.T) { signer, _ := newTestSigner() - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // add the data our multihash aliased manifest will point to @@ -208,7 +210,7 @@ func TestBzzResourceMultihash(t *testing.T) { // Test resource updates using the raw update methods func TestBzzResource(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) signer, _ := newTestSigner() defer srv.Close() @@ -467,7 +469,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { addr := [3]storage.Address{} - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() for i, mf := range testmanifest { @@ -702,7 +704,7 @@ func TestBzzTar(t *testing.T) { } func testBzzTar(encrypted bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() fileNames := []string{"tmp1.txt", "tmp2.lock", "tmp3.rtf"} fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} @@ -827,7 +829,7 @@ func TestBzzRootRedirectEncrypted(t *testing.T) { } func testBzzRootRedirect(toEncrypt bool, t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() // create a manifest with some data at the root path @@ -882,7 +884,7 @@ func testBzzRootRedirect(toEncrypt bool, t *testing.T) { } func TestMethodsNotAllowed(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() databytes := "bar" for _, c := range []struct { @@ -941,7 +943,7 @@ func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string } func TestGet(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() for _, testCase := range []struct { @@ -1025,7 +1027,7 @@ func TestGet(t *testing.T) { } func TestModify(t *testing.T) { - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() swarmClient := swarm.NewClient(srv.URL) @@ -1126,7 +1128,7 @@ func TestMultiPartUpload(t *testing.T) { // POST /bzz:/ Content-Type: multipart/form-data verbose := false // Setup Swarm - srv := testutil.NewTestSwarmServer(t, serverFunc) + srv := testutil.NewTestSwarmServer(t, serverFunc, nil) defer srv.Close() url := fmt.Sprintf("%s/bzz:/", srv.URL) @@ -1153,3 +1155,126 @@ func TestMultiPartUpload(t *testing.T) { t.Fatalf("expected POST multipart/form-data to return a 64 char manifest but the answer was %d chars long", len(body)) } } + +// TestBzzGetFileWithResolver tests fetching a file using a mocked ENS resolver +func TestBzzGetFileWithResolver(t *testing.T) { + resolver := newTestResolveValidator("") + srv := testutil.NewTestSwarmServer(t, serverFunc, resolver) + defer srv.Close() + fileNames := []string{"dir1/tmp1.txt", "dir2/tmp2.lock", "dir3/tmp3.rtf"} + fileContents := []string{"tmp1textfilevalue", "tmp2lockfilelocked", "tmp3isjustaplaintextfile"} + + buf := &bytes.Buffer{} + tw := tar.NewWriter(buf) + + for i, v := range fileNames { + size := len(fileContents[i]) + hdr := &tar.Header{ + Name: v, + Mode: 0644, + Size: int64(size), + ModTime: time.Now(), + Xattrs: map[string]string{ + "user.swarm.content-type": "text/plain", + }, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatal(err) + } + + // copy the file into the tar stream + n, err := io.WriteString(tw, fileContents[i]) + if err != nil { + t.Fatal(err) + } else if n != size { + t.Fatal("size mismatch") + } + } + + if err := tw.Close(); err != nil { + t.Fatal(err) + } + + //post tar stream + url := srv.URL + "/bzz:/" + + req, err := http.NewRequest("POST", url, buf) + if err != nil { + t.Fatal(err) + } + req.Header.Add("Content-Type", "application/x-tar") + client := &http.Client{} + serverResponse, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + if serverResponse.StatusCode != http.StatusOK { + t.Fatalf("err %s", serverResponse.Status) + } + swarmHash, err := ioutil.ReadAll(serverResponse.Body) + serverResponse.Body.Close() + if err != nil { + t.Fatal(err) + } + // set the resolved hash to be the swarm hash of what we've just uploaded + hash := common.HexToHash(string(swarmHash)) + resolver.hash = &hash + for _, v := range []struct { + addr string + path string + expectedStatusCode int + }{ + { + addr: string(swarmHash), + path: fileNames[0], + expectedStatusCode: http.StatusOK, + }, + { + addr: "somebogusensname", + path: fileNames[0], + expectedStatusCode: http.StatusOK, + }, + } { + req, err := http.NewRequest("GET", fmt.Sprintf(srv.URL+"/bzz:/%s/%s", v.addr, v.path), nil) + if err != nil { + t.Fatal(err) + } + serverResponse, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer serverResponse.Body.Close() + if serverResponse.StatusCode != v.expectedStatusCode { + t.Fatalf("expected %d, got %d", v.expectedStatusCode, serverResponse.StatusCode) + } + } +} + +// testResolver implements the Resolver interface and either returns the given +// hash if it is set, or returns a "name not found" error +type testResolveValidator struct { + hash *common.Hash +} + +func newTestResolveValidator(addr string) *testResolveValidator { + r := &testResolveValidator{} + if addr != "" { + hash := common.HexToHash(addr) + r.hash = &hash + } + return r +} + +func (t *testResolveValidator) Resolve(addr string) (common.Hash, error) { + if t.hash == nil { + return common.Hash{}, fmt.Errorf("DNS name not found: %q", addr) + } + return *t.hash, nil +} + +func (t *testResolveValidator) Owner(node [32]byte) (addr common.Address, err error) { + return +} +func (t *testResolveValidator) HeaderByNumber(context.Context, *big.Int) (header *types.Header, err error) { + return +} diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 7fd60fcc3d..0748230329 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -45,7 +45,7 @@ func (f *fakeTimeProvider) Now() mru.Timestamp { return mru.Timestamp{Time: f.currentTime} } -func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer) *TestSwarmServer { +func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer, resolver api.Resolver) *TestSwarmServer { dir, err := ioutil.TempDir("", "swarm-storage-test") if err != nil { t.Fatal(err) @@ -77,7 +77,7 @@ func NewTestSwarmServer(t *testing.T, serverFunc func(*api.API) TestServer) *Tes t.Fatal(err) } - a := api.NewAPI(fileStore, nil, rh.Handler, nil) + a := api.NewAPI(fileStore, resolver, rh.Handler, nil) srv := httptest.NewServer(serverFunc(a)) return &TestSwarmServer{ Server: srv, From 580145e96db848cb8e2f8bb8f0621bcacbc9521c Mon Sep 17 00:00:00 2001 From: Elad Date: Thu, 6 Sep 2018 12:11:38 +0200 Subject: [PATCH 89/96] swarm/storage: added metrics for db entry count (#17589) --- swarm/api/http/server_test.go | 39 +++++++++++++++++------------------ swarm/storage/ldbstore.go | 6 ++++++ 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 891deb8817..2b6a97ebdf 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -893,14 +893,14 @@ func TestMethodsNotAllowed(t *testing.T) { }{ { url: fmt.Sprintf("%s/bzz-list:/", srv.URL), - code: 405, + code: http.StatusMethodNotAllowed, }, { url: fmt.Sprintf("%s/bzz-hash:/", srv.URL), - code: 405, + code: http.StatusMethodNotAllowed, }, { url: fmt.Sprintf("%s/bzz-immutable:/", srv.URL), - code: 405, + code: http.StatusMethodNotAllowed, }, } { res, _ := http.Post(c.url, "text/plain", bytes.NewReader([]byte(databytes))) @@ -958,7 +958,7 @@ func TestGet(t *testing.T) { uri: fmt.Sprintf("%s/", srv.URL), method: "GET", headers: map[string]string{"Accept": "text/html"}, - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseBody: "Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution", verbose: false, }, @@ -966,7 +966,7 @@ func TestGet(t *testing.T) { uri: fmt.Sprintf("%s/", srv.URL), method: "GET", headers: map[string]string{"Accept": "application/json"}, - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseBody: "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", verbose: false, }, @@ -974,7 +974,7 @@ func TestGet(t *testing.T) { uri: fmt.Sprintf("%s/robots.txt", srv.URL), method: "GET", headers: map[string]string{"Accept": "text/html"}, - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseBody: "User-agent: *\nDisallow: /", verbose: false, }, @@ -982,38 +982,37 @@ func TestGet(t *testing.T) { uri: fmt.Sprintf("%s/nonexistent_path", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 404, + expectedStatusCode: http.StatusNotFound, verbose: false, }, { uri: fmt.Sprintf("%s/bzz:asdf/", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 404, + expectedStatusCode: http.StatusNotFound, verbose: false, }, { uri: fmt.Sprintf("%s/tbz2/", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 404, + expectedStatusCode: http.StatusNotFound, verbose: false, }, { uri: fmt.Sprintf("%s/bzz-rack:/", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 404, + expectedStatusCode: http.StatusNotFound, verbose: false, }, { uri: fmt.Sprintf("%s/bzz-ls", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 404, + expectedStatusCode: http.StatusNotFound, verbose: false, - }, - } { + }} { t.Run("GET "+testCase.uri, func(t *testing.T) { res, body := httpDo(testCase.method, testCase.uri, nil, testCase.headers, testCase.verbose, t) if res.StatusCode != testCase.expectedStatusCode { @@ -1060,7 +1059,7 @@ func TestModify(t *testing.T) { uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), method: "DELETE", headers: map[string]string{}, - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseBody: "8b634aea26eec353ac0ecbec20c94f44d6f8d11f38d4578a4c207a84c74ef731", verbose: false, }, @@ -1068,21 +1067,21 @@ func TestModify(t *testing.T) { uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), method: "PUT", headers: map[string]string{}, - expectedStatusCode: 405, + expectedStatusCode: http.StatusMethodNotAllowed, verbose: false, }, { uri: fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, hash), method: "PUT", headers: map[string]string{}, - expectedStatusCode: 405, + expectedStatusCode: http.StatusMethodNotAllowed, verbose: false, }, { uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), method: "PATCH", headers: map[string]string{}, - expectedStatusCode: 405, + expectedStatusCode: http.StatusMethodNotAllowed, verbose: false, }, { @@ -1090,7 +1089,7 @@ func TestModify(t *testing.T) { method: "POST", headers: map[string]string{}, requestBody: []byte("POSTdata"), - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseHeaders: map[string]string{"Content-Length": "64"}, verbose: false, }, @@ -1099,7 +1098,7 @@ func TestModify(t *testing.T) { method: "POST", headers: map[string]string{}, requestBody: []byte("POSTdata"), - expectedStatusCode: 200, + expectedStatusCode: http.StatusOK, assertResponseHeaders: map[string]string{"Content-Length": "128"}, verbose: false, }, @@ -1148,7 +1147,7 @@ func TestMultiPartUpload(t *testing.T) { } res, body := httpDo("POST", url, buf, headers, verbose, t) - if res.StatusCode != 200 { + if res.StatusCode != http.StatusOK { t.Fatalf("expected POST multipart/form-data to return 200, but it returned %d", res.StatusCode) } if len(body) != 64 { diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index b95aa13b09..bd3501ea26 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -48,6 +48,10 @@ const ( maxGCitems = 5000 // max number of items to be gc'd per call to collectGarbage() ) +var ( + dbEntryCount = metrics.NewRegisteredCounter("ldbstore.entryCnt", nil) +) + var ( keyIndex = byte(0) keyOldData = byte(1) @@ -495,6 +499,7 @@ func (s *LDBStore) delete(idx uint64, idxKey []byte, po uint8) { batch.Delete(idxKey) batch.Delete(getDataKey(idx, po)) s.entryCnt-- + dbEntryCount.Dec(1) s.bucketCnt[po]-- cntKey := make([]byte, 2) cntKey[0] = keyDistanceCnt @@ -566,6 +571,7 @@ func (s *LDBStore) doPut(chunk *Chunk, index *dpaDBIndex, po uint8) { index.Idx = s.dataIdx s.bucketCnt[po] = s.dataIdx s.entryCnt++ + dbEntryCount.Inc(1) s.dataIdx++ cntKey := make([]byte, 2) From 70d31fb27842b047582a6557529b2234de1a4a8d Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 7 Sep 2018 09:56:05 +0200 Subject: [PATCH 90/96] cmd/swarm: added password to ACT (#17598) --- cmd/swarm/access.go | 49 +++++++---- cmd/swarm/access_test.go | 176 +++++++++++++++------------------------ cmd/swarm/main.go | 1 + swarm/api/act.go | 149 ++++++++++++++++++++++++--------- swarm/testutil/file.go | 44 ++++++++++ 5 files changed, 250 insertions(+), 169 deletions(-) create mode 100644 swarm/testutil/file.go diff --git a/cmd/swarm/access.go b/cmd/swarm/access.go index 12cfbfc1a4..1e69526ec3 100644 --- a/cmd/swarm/access.go +++ b/cmd/swarm/access.go @@ -51,7 +51,7 @@ func accessNewPass(ctx *cli.Context) { password = getPassPhrase("", 0, makePasswordList(ctx)) dryRun = ctx.Bool(SwarmDryRunFlag.Name) ) - accessKey, ae, err = api.DoPasswordNew(ctx, password, salt) + accessKey, ae, err = api.DoPassword(ctx, password, salt) if err != nil { utils.Fatalf("error getting session key: %v", err) } @@ -85,7 +85,7 @@ func accessNewPK(ctx *cli.Context) { granteePublicKey = ctx.String(SwarmAccessGrantKeyFlag.Name) dryRun = ctx.Bool(SwarmDryRunFlag.Name) ) - sessionKey, ae, err = api.DoPKNew(ctx, privateKey, granteePublicKey, salt) + sessionKey, ae, err = api.DoPK(ctx, privateKey, granteePublicKey, salt) if err != nil { utils.Fatalf("error getting session key: %v", err) } @@ -110,23 +110,38 @@ func accessNewACT(ctx *cli.Context) { } var ( - ae *api.AccessEntry - actManifest *api.Manifest - accessKey []byte - err error - ref = args[0] - grantees = []string{} - actFilename = ctx.String(SwarmAccessGrantKeysFlag.Name) - privateKey = getPrivKey(ctx) - dryRun = ctx.Bool(SwarmDryRunFlag.Name) + ae *api.AccessEntry + actManifest *api.Manifest + accessKey []byte + err error + ref = args[0] + pkGrantees = []string{} + passGrantees = []string{} + pkGranteesFilename = ctx.String(SwarmAccessGrantKeysFlag.Name) + passGranteesFilename = ctx.String(utils.PasswordFileFlag.Name) + privateKey = getPrivKey(ctx) + dryRun = ctx.Bool(SwarmDryRunFlag.Name) ) - - bytes, err := ioutil.ReadFile(actFilename) - if err != nil { - utils.Fatalf("had an error reading the grantee public key list") + if pkGranteesFilename == "" && passGranteesFilename == "" { + utils.Fatalf("you have to provide either a grantee public-keys file or an encryption passwords file (or both)") } - grantees = strings.Split(string(bytes), "\n") - accessKey, ae, actManifest, err = api.DoACTNew(ctx, privateKey, salt, grantees) + + if pkGranteesFilename != "" { + bytes, err := ioutil.ReadFile(pkGranteesFilename) + if err != nil { + utils.Fatalf("had an error reading the grantee public key list") + } + pkGrantees = strings.Split(string(bytes), "\n") + } + + if passGranteesFilename != "" { + bytes, err := ioutil.ReadFile(passGranteesFilename) + if err != nil { + utils.Fatalf("could not read password filename: %v", err) + } + passGrantees = strings.Split(string(bytes), "\n") + } + accessKey, ae, actManifest, err = api.DoACT(ctx, privateKey, salt, pkGrantees, passGrantees) if err != nil { utils.Fatalf("error generating ACT manifest: %v", err) } diff --git a/cmd/swarm/access_test.go b/cmd/swarm/access_test.go index 384d256306..106e6dd912 100644 --- a/cmd/swarm/access_test.go +++ b/cmd/swarm/access_test.go @@ -28,7 +28,6 @@ import ( gorand "math/rand" "net/http" "os" - "path/filepath" "strings" "testing" "time" @@ -39,6 +38,12 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/swarm/testutil" +) + +const ( + hashRegexp = `[a-f\d]{128}` + data = "notsorandomdata" ) var DefaultCurve = crypto.S256() @@ -53,23 +58,8 @@ func TestAccessPassword(t *testing.T) { defer cluster.Shutdown() proxyNode := cluster.Nodes[0] - // create a tmp file - tmp, err := ioutil.TempDir("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmp) - - // write data to file - data := "notsorandomdata" - dataFilename := filepath.Join(tmp, "data.txt") - - err = ioutil.WriteFile(dataFilename, []byte(data), 0666) - if err != nil { - t.Fatal(err) - } - - hashRegexp := `[a-f\d]{128}` + dataFilename := testutil.TempFileWithContent(t, data) + defer os.RemoveAll(dataFilename) // upload the file with 'swarm up' and expect a hash up := runSwarm(t, @@ -86,14 +76,14 @@ func TestAccessPassword(t *testing.T) { } ref := matches[0] - - password := "smth" - passwordFilename := filepath.Join(tmp, "password.txt") - - err = ioutil.WriteFile(passwordFilename, []byte(password), 0666) + tmp, err := ioutil.TempDir("", "swarm-test") if err != nil { t.Fatal(err) } + defer os.RemoveAll(tmp) + password := "smth" + passwordFilename := testutil.TempFileWithContent(t, "smth") + defer os.RemoveAll(passwordFilename) up = runSwarm(t, "access", @@ -193,12 +183,8 @@ func TestAccessPassword(t *testing.T) { t.Errorf("expected decrypted data %q, got %q", data, string(d)) } - wrongPasswordFilename := filepath.Join(tmp, "password-wrong.txt") - - err = ioutil.WriteFile(wrongPasswordFilename, []byte("just wr0ng"), 0666) - if err != nil { - t.Fatal(err) - } + wrongPasswordFilename := testutil.TempFileWithContent(t, "just wr0ng") + defer os.RemoveAll(wrongPasswordFilename) //download file with 'swarm down' with wrong password up = runSwarm(t, @@ -227,22 +213,8 @@ func TestAccessPK(t *testing.T) { cluster := newTestCluster(t, 2) defer cluster.Shutdown() - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer tmp.Close() - defer os.Remove(tmp.Name()) - - // write data to file - data := "notsorandomdata" - _, err = io.WriteString(tmp, data) - if err != nil { - t.Fatal(err) - } - - hashRegexp := `[a-f\d]{128}` + dataFilename := testutil.TempFileWithContent(t, data) + defer os.RemoveAll(dataFilename) // upload the file with 'swarm up' and expect a hash up := runSwarm(t, @@ -250,7 +222,7 @@ func TestAccessPK(t *testing.T) { cluster.Nodes[0].URL, "up", "--encrypt", - tmp.Name()) + dataFilename) _, matches := up.ExpectRegexp(hashRegexp) up.ExpectExit() @@ -259,7 +231,6 @@ func TestAccessPK(t *testing.T) { } ref := matches[0] - pk := cluster.Nodes[0].PrivateKey granteePubKey := crypto.CompressPubkey(&pk.PublicKey) @@ -268,22 +239,15 @@ func TestAccessPK(t *testing.T) { t.Fatal(err) } - passFile, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer passFile.Close() - defer os.Remove(passFile.Name()) - _, err = io.WriteString(passFile, testPassphrase) - if err != nil { - t.Fatal(err) - } + passwordFilename := testutil.TempFileWithContent(t, testPassphrase) + defer os.RemoveAll(passwordFilename) + _, publisherAccount := getTestAccount(t, publisherDir) up = runSwarm(t, "--bzzaccount", publisherAccount.Address.String(), "--password", - passFile.Name(), + passwordFilename, "--datadir", publisherDir, "--bzzapi", @@ -309,7 +273,7 @@ func TestAccessPK(t *testing.T) { "--bzzaccount", publisherAccount.Address.String(), "--password", - passFile.Name(), + passwordFilename, "--datadir", publisherDir, "print-keys", @@ -390,37 +354,24 @@ func TestAccessACTScale(t *testing.T) { testAccessACT(t, 1000) } -// TestAccessACT tests the e2e creation, uploading and downloading of an ACT type access control +// TestAccessACT tests the e2e creation, uploading and downloading of an ACT access control with both EC keys AND password protection // the test fires up a 3 node cluster, then randomly picks 2 nodes which will be acting as grantees to the data -// set. the third node should fail decoding the reference as it will not be granted access. the publisher uploads through -// one of the nodes then disappears. If `bogusEntries` is bigger than 0, the test will generate the number of bogus act entries -// to test what happens at scale +// set and also protects the ACT with a password. the third node should fail decoding the reference as it will not be granted access. +// the third node then then tries to download using a correct password (and succeeds) then uses a wrong password and fails. +// the publisher uploads through one of the nodes then disappears. func testAccessACT(t *testing.T, bogusEntries int) { // Setup Swarm and upload a test file to it - cluster := newTestCluster(t, 3) + const clusterSize = 3 + cluster := newTestCluster(t, clusterSize) defer cluster.Shutdown() var uploadThroughNode = cluster.Nodes[0] client := swarm.NewClient(uploadThroughNode.URL) r1 := gorand.New(gorand.NewSource(time.Now().UnixNano())) - nodeToSkip := r1.Intn(3) // a number between 0 and 2 (node indices in `cluster`) - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer tmp.Close() - defer os.Remove(tmp.Name()) - - // write data to file - data := "notsorandomdata" - _, err = io.WriteString(tmp, data) - if err != nil { - t.Fatal(err) - } - - hashRegexp := `[a-f\d]{128}` + nodeToSkip := r1.Intn(clusterSize) // a number between 0 and 2 (node indices in `cluster`) + dataFilename := testutil.TempFileWithContent(t, data) + defer os.RemoveAll(dataFilename) // upload the file with 'swarm up' and expect a hash up := runSwarm(t, @@ -428,7 +379,7 @@ func testAccessACT(t *testing.T, bogusEntries int) { cluster.Nodes[0].URL, "up", "--encrypt", - tmp.Name()) + dataFilename) _, matches := up.ExpectRegexp(hashRegexp) up.ExpectExit() @@ -464,41 +415,25 @@ func testAccessACT(t *testing.T, bogusEntries int) { } grantees = bogusGrantees } - - granteesPubkeyListFile, err := ioutil.TempFile("", "grantees-pubkey-list") - if err != nil { - t.Fatal(err) - } - defer granteesPubkeyListFile.Close() - defer os.Remove(granteesPubkeyListFile.Name()) - - _, err = granteesPubkeyListFile.WriteString(strings.Join(grantees, "\n")) - if err != nil { - t.Fatal(err) - } + granteesPubkeyListFile := testutil.TempFileWithContent(t, strings.Join(grantees, "\n")) + defer os.RemoveAll(granteesPubkeyListFile) publisherDir, err := ioutil.TempDir("", "swarm-account-dir-temp") if err != nil { t.Fatal(err) } + defer os.RemoveAll(publisherDir) - passFile, err := ioutil.TempFile("", "swarm-test") - if err != nil { - t.Fatal(err) - } - defer passFile.Close() - defer os.Remove(passFile.Name()) - _, err = io.WriteString(passFile, testPassphrase) - if err != nil { - t.Fatal(err) - } - + passwordFilename := testutil.TempFileWithContent(t, testPassphrase) + defer os.RemoveAll(passwordFilename) + actPasswordFilename := testutil.TempFileWithContent(t, "smth") + defer os.RemoveAll(actPasswordFilename) _, publisherAccount := getTestAccount(t, publisherDir) up = runSwarm(t, "--bzzaccount", publisherAccount.Address.String(), "--password", - passFile.Name(), + passwordFilename, "--datadir", publisherDir, "--bzzapi", @@ -507,7 +442,9 @@ func testAccessACT(t *testing.T, bogusEntries int) { "new", "act", "--grant-keys", - granteesPubkeyListFile.Name(), + granteesPubkeyListFile, + "--password", + actPasswordFilename, ref, ) @@ -523,7 +460,7 @@ func testAccessACT(t *testing.T, bogusEntries int) { "--bzzaccount", publisherAccount.Address.String(), "--password", - passFile.Name(), + passwordFilename, "--datadir", publisherDir, "print-keys", @@ -562,9 +499,7 @@ func testAccessACT(t *testing.T, bogusEntries int) { if len(a.Salt) < 32 { t.Fatalf(`got salt with length %v, expected not less the 32 bytes`, len(a.Salt)) } - if a.KdfParams != nil { - t.Fatal("manifest access kdf params should be nil") - } + if a.Publisher != pkComp { t.Fatal("publisher key did not match") } @@ -588,6 +523,25 @@ func testAccessACT(t *testing.T, bogusEntries int) { t.Fatalf("should be a 401") } + // try downloading using a password instead, using the unauthorized node + passwordUrl := strings.Replace(url, "http://", "http://:smth@", -1) + response, err = httpClient.Get(passwordUrl) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusOK { + t.Fatal("should be a 200") + } + + // now try with the wrong password, expect 401 + passwordUrl = strings.Replace(url, "http://", "http://:smthWrong@", -1) + response, err = httpClient.Get(passwordUrl) + if err != nil { + t.Fatal(err) + } + if response.StatusCode != http.StatusUnauthorized { + t.Fatal("should be a 401") + } continue } diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index e654409371..c93344c420 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -319,6 +319,7 @@ func init() { Flags: []cli.Flag{ SwarmAccessGrantKeysFlag, SwarmDryRunFlag, + utils.PasswordFileFlag, }, Name: "act", Usage: "encrypts a reference with the node's private key and a given grantee's public key and embeds it into a root manifest", diff --git a/swarm/api/act.go b/swarm/api/act.go index b1a5947831..52d9098271 100644 --- a/swarm/api/act.go +++ b/swarm/api/act.go @@ -102,6 +102,7 @@ const AccessTypePass = AccessType("pass") const AccessTypePK = AccessType("pk") const AccessTypeACT = AccessType("act") +// NewAccessEntryPassword creates a manifest AccessEntry in order to create an ACT protected by a password func NewAccessEntryPassword(salt []byte, kdfParams *KdfParams) (*AccessEntry, error) { if len(salt) != 32 { return nil, fmt.Errorf("salt should be 32 bytes long") @@ -113,6 +114,7 @@ func NewAccessEntryPassword(salt []byte, kdfParams *KdfParams) (*AccessEntry, er }, nil } +// NewAccessEntryPK creates a manifest AccessEntry in order to create an ACT protected by a pair of Elliptic Curve keys func NewAccessEntryPK(publisher string, salt []byte) (*AccessEntry, error) { if len(publisher) != 66 { return nil, fmt.Errorf("publisher should be 66 characters long, got %d", len(publisher)) @@ -127,6 +129,7 @@ func NewAccessEntryPK(publisher string, salt []byte) (*AccessEntry, error) { }, nil } +// NewAccessEntryACT creates a manifest AccessEntry in order to create an ACT protected by a combination of EC keys and passwords func NewAccessEntryACT(publisher string, salt []byte, act string) (*AccessEntry, error) { if len(salt) != 32 { return nil, fmt.Errorf("salt should be 32 bytes long") @@ -140,15 +143,19 @@ func NewAccessEntryACT(publisher string, salt []byte, act string) (*AccessEntry, Publisher: publisher, Salt: salt, Act: act, + KdfParams: DefaultKdfParams, }, nil } +// NOOPDecrypt is a generic decrypt function that is passed into the API in places where real ACT decryption capabilities are +// either unwanted, or alternatively, cannot be implemented in the immediate scope func NOOPDecrypt(*ManifestEntry) error { return nil } var DefaultKdfParams = NewKdfParams(262144, 1, 8) +// NewKdfParams returns a KdfParams struct with the given scrypt params func NewKdfParams(n, p, r int) *KdfParams { return &KdfParams{ @@ -161,15 +168,20 @@ func NewKdfParams(n, p, r int) *KdfParams { // NewSessionKeyPassword creates a session key based on a shared secret (password) and the given salt // and kdf parameters in the access entry func NewSessionKeyPassword(password string, accessEntry *AccessEntry) ([]byte, error) { - if accessEntry.Type != AccessTypePass { + if accessEntry.Type != AccessTypePass && accessEntry.Type != AccessTypeACT { return nil, errors.New("incorrect access entry type") + } + return sessionKeyPassword(password, accessEntry.Salt, accessEntry.KdfParams) +} + +func sessionKeyPassword(password string, salt []byte, kdfParams *KdfParams) ([]byte, error) { return scrypt.Key( []byte(password), - accessEntry.Salt, - accessEntry.KdfParams.N, - accessEntry.KdfParams.R, - accessEntry.KdfParams.P, + salt, + kdfParams.N, + kdfParams.R, + kdfParams.P, 32, ) } @@ -188,9 +200,6 @@ func NewSessionKeyPK(private *ecdsa.PrivateKey, public *ecdsa.PublicKey, salt [] return sessionKey, nil } -func (a *API) NodeSessionKey(privateKey *ecdsa.PrivateKey, publicKey *ecdsa.PublicKey, salt []byte) ([]byte, error) { - return NewSessionKeyPK(privateKey, publicKey, salt) -} func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.PrivateKey) DecryptFunc { return func(m *ManifestEntry) error { if m.Access == nil { @@ -242,7 +251,7 @@ func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.Priva if err != nil { return ErrDecrypt } - key, err := a.NodeSessionKey(pk, publisher, m.Access.Salt) + key, err := NewSessionKeyPK(pk, publisher, m.Access.Salt) if err != nil { return ErrDecrypt } @@ -261,6 +270,11 @@ func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.Priva m.Access = nil return nil case "act": + var ( + sessionKey []byte + err error + ) + publisherBytes, err := hex.DecodeString(m.Access.Publisher) if err != nil { return ErrDecrypt @@ -270,40 +284,35 @@ func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.Priva return ErrDecrypt } - sessionKey, err := a.NodeSessionKey(pk, publisher, m.Access.Salt) + sessionKey, err = NewSessionKeyPK(pk, publisher, m.Access.Salt) if err != nil { return ErrDecrypt } - hasher := sha3.NewKeccak256() - hasher.Write(append(sessionKey, 0)) - lookupKey := hasher.Sum(nil) - - hasher.Reset() - - hasher.Write(append(sessionKey, 1)) - accessKeyDecryptionKey := hasher.Sum(nil) - - lk := hex.EncodeToString(lookupKey) - list, err := a.GetManifestList(ctx, NOOPDecrypt, storage.Address(common.Hex2Bytes(m.Access.Act)), lk) - - found := "" - for _, v := range list.Entries { - if v.Path == lk { - found = v.Hash - } - } - - if found == "" { - return ErrDecrypt - } - - v, err := hex.DecodeString(found) + found, ciphertext, decryptionKey, err := a.getACTDecryptionKey(ctx, storage.Address(common.Hex2Bytes(m.Access.Act)), sessionKey) if err != nil { return err } - enc := NewRefEncryption(len(v) - 8) - decodedRef, err := enc.Decrypt(v, accessKeyDecryptionKey) + if !found { + // try to fall back to password + if credentials != "" { + sessionKey, err = NewSessionKeyPassword(credentials, m.Access) + if err != nil { + return err + } + found, ciphertext, decryptionKey, err = a.getACTDecryptionKey(ctx, storage.Address(common.Hex2Bytes(m.Access.Act)), sessionKey) + if err != nil { + return err + } + if !found { + return ErrDecrypt + } + } else { + return ErrDecrypt + } + } + enc := NewRefEncryption(len(ciphertext) - 8) + decodedRef, err := enc.Decrypt(ciphertext, decryptionKey) if err != nil { return ErrDecrypt } @@ -326,6 +335,33 @@ func (a *API) doDecrypt(ctx context.Context, credentials string, pk *ecdsa.Priva } } +func (a *API) getACTDecryptionKey(ctx context.Context, actManifestAddress storage.Address, sessionKey []byte) (found bool, ciphertext, decryptionKey []byte, err error) { + hasher := sha3.NewKeccak256() + hasher.Write(append(sessionKey, 0)) + lookupKey := hasher.Sum(nil) + hasher.Reset() + + hasher.Write(append(sessionKey, 1)) + accessKeyDecryptionKey := hasher.Sum(nil) + hasher.Reset() + + lk := hex.EncodeToString(lookupKey) + list, err := a.GetManifestList(ctx, NOOPDecrypt, actManifestAddress, lk) + if err != nil { + return false, nil, nil, err + } + for _, v := range list.Entries { + if v.Path == lk { + cipherTextBytes, err := hex.DecodeString(v.Hash) + if err != nil { + return false, nil, nil, err + } + return true, cipherTextBytes, accessKeyDecryptionKey, nil + } + } + return false, nil, nil, nil +} + func GenerateAccessControlManifest(ctx *cli.Context, ref string, accessKey []byte, ae *AccessEntry) (*Manifest, error) { refBytes, err := hex.DecodeString(ref) if err != nil { @@ -352,7 +388,9 @@ func GenerateAccessControlManifest(ctx *cli.Context, ref string, accessKey []byt return m, nil } -func DoPKNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { +// DoPK is a helper function to the CLI API that handles the entire business logic for +// creating a session key and access entry given the cli context, ec keys and salt +func DoPK(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { if granteePublicKey == "" { return nil, nil, errors.New("need a grantee Public Key") } @@ -383,9 +421,11 @@ func DoPKNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, granteePublicKey st return sessionKey, ae, nil } -func DoACTNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees []string) (accessKey []byte, ae *AccessEntry, actManifest *Manifest, err error) { - if len(grantees) == 0 { - return nil, nil, nil, errors.New("did not get any grantee public keys") +// DoACT is a helper function to the CLI API that handles the entire business logic for +// creating a access key, access entry and ACT manifest (including uploading it) given the cli context, ec keys, password grantees and salt +func DoACT(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grantees []string, encryptPasswords []string) (accessKey []byte, ae *AccessEntry, actManifest *Manifest, err error) { + if len(grantees) == 0 && len(encryptPasswords) == 0 { + return nil, nil, nil, errors.New("did not get any grantee public keys or any encryption passwords") } publisherPub := hex.EncodeToString(crypto.CompressPubkey(&privateKey.PublicKey)) @@ -430,7 +470,31 @@ func DoACTNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grant enc := NewRefEncryption(len(accessKey)) encryptedAccessKey, err := enc.Encrypt(accessKey, accessKeyEncryptionKey) + if err != nil { + return nil, nil, nil, err + } + lookupPathEncryptedAccessKeyMap[hex.EncodeToString(lookupKey)] = hex.EncodeToString(encryptedAccessKey) + } + for _, pass := range encryptPasswords { + sessionKey, err := sessionKeyPassword(pass, salt, DefaultKdfParams) + if err != nil { + return nil, nil, nil, err + } + hasher := sha3.NewKeccak256() + hasher.Write(append(sessionKey, 0)) + lookupKey := hasher.Sum(nil) + + hasher.Reset() + hasher.Write(append(sessionKey, 1)) + + accessKeyEncryptionKey := hasher.Sum(nil) + + enc := NewRefEncryption(len(accessKey)) + encryptedAccessKey, err := enc.Encrypt(accessKey, accessKeyEncryptionKey) + if err != nil { + return nil, nil, nil, err + } lookupPathEncryptedAccessKeyMap[hex.EncodeToString(lookupKey)] = hex.EncodeToString(encryptedAccessKey) } @@ -454,7 +518,10 @@ func DoACTNew(ctx *cli.Context, privateKey *ecdsa.PrivateKey, salt []byte, grant return accessKey, ae, m, nil } -func DoPasswordNew(ctx *cli.Context, password string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { +// DoPassword is a helper function to the CLI API that handles the entire business logic for +// creating a session key and an access entry given the cli context, password and salt. +// By default - DefaultKdfParams are used as the scrypt params +func DoPassword(ctx *cli.Context, password string, salt []byte) (sessionKey []byte, ae *AccessEntry, err error) { ae, err = NewAccessEntryPassword(salt, DefaultKdfParams) if err != nil { return nil, nil, err diff --git a/swarm/testutil/file.go b/swarm/testutil/file.go new file mode 100644 index 0000000000..ecb0d971ed --- /dev/null +++ b/swarm/testutil/file.go @@ -0,0 +1,44 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package testutil + +import ( + "io" + "io/ioutil" + "os" + "strings" + "testing" +) + +// TempFileWithContent is a helper function that creates a temp file that contains the following string content then closes the file handle +// it returns the complete file path +func TempFileWithContent(t *testing.T, content string) string { + tempFile, err := ioutil.TempFile("", "swarm-temp-file") + if err != nil { + t.Fatal(err) + } + + _, err = io.Copy(tempFile, strings.NewReader(content)) + if err != nil { + os.RemoveAll(tempFile.Name()) + t.Fatal(err) + } + if err = tempFile.Close(); err != nil { + t.Fatal(err) + } + return tempFile.Name() +} From 8b9b149d5412dd3c775caf4fc3df39ebad90184f Mon Sep 17 00:00:00 2001 From: Elad Date: Fri, 7 Sep 2018 15:23:29 +0200 Subject: [PATCH 91/96] swarm/api/http: bzz-immutable wrong handler bug (#17602) --- swarm/api/http/server.go | 2 +- swarm/api/http/server_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 2aa1963969..af1269b934 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -129,7 +129,7 @@ func NewServer(api *api.API, corsString string) *Server { }) mux.Handle("/bzz-immutable:/", methodHandler{ "GET": Adapt( - http.HandlerFunc(server.HandleGet), + http.HandlerFunc(server.HandleBzzGet), defaultMiddlewares..., ), }) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 2b6a97ebdf..efefa9fae4 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -672,7 +672,7 @@ func testBzzGetPath(encrypted bool, t *testing.T) { nonhashresponses := []string{ `cannot resolve name: no DNS to resolve name: "name"`, - `cannot resolve nonhash: immutable address not a content hash: "nonhash"`, + `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, `cannot resolve nonhash: no DNS to resolve name: "nonhash"`, From ae992a5d73311742389fce3f855575be98fc6972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Fri, 7 Sep 2018 18:13:25 +0200 Subject: [PATCH 92/96] core/vm: Hide read only flag from Interpreter interface (#17461) Makes Interface interface a bit more stateless and abstract. Obviously this change is dictated by EVMC design. The EVMC tries to keep the responsibility for EVM features totally inside the VMs, if feasible. This makes VM "stateless" because VM does not need to pass any information between executions, all information is included in parameters of the execute function. --- core/vm/evm.go | 21 +++++++-------------- core/vm/interpreter.go | 25 +++++++++---------------- eth/tracers/tracer_test.go | 2 +- 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index a24f6f3865..58618f8115 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -41,7 +41,7 @@ type ( ) // run runs the given contract and takes care of running precompiles with a fallback to the byte code interpreter. -func run(evm *EVM, contract *Contract, input []byte) ([]byte, error) { +func run(evm *EVM, contract *Contract, input []byte, readOnly bool) ([]byte, error) { if contract.CodeAddr != nil { precompiles := PrecompiledContractsHomestead if evm.ChainConfig().IsByzantium(evm.BlockNumber) { @@ -61,7 +61,7 @@ func run(evm *EVM, contract *Contract, input []byte) ([]byte, error) { }(evm.interpreter) evm.interpreter = interpreter } - return interpreter.Run(contract, input) + return interpreter.Run(contract, input, readOnly) } } return nil, ErrNoCompatibleInterpreter @@ -210,7 +210,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas evm.vmConfig.Tracer.CaptureEnd(ret, gas-contract.Gas, time.Since(start), err) }() } - ret, err = run(evm, contract, input) + ret, err = run(evm, contract, input, false) // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally @@ -255,7 +255,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, contract := NewContract(caller, to, value, gas) contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) - ret, err = run(evm, contract, input) + ret, err = run(evm, contract, input, false) if err != nil { evm.StateDB.RevertToSnapshot(snapshot) if err != errExecutionReverted { @@ -288,7 +288,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by contract := NewContract(caller, to, nil, gas).AsDelegate() contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) - ret, err = run(evm, contract, input) + ret, err = run(evm, contract, input, false) if err != nil { evm.StateDB.RevertToSnapshot(snapshot) if err != errExecutionReverted { @@ -310,13 +310,6 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte if evm.depth > int(params.CallCreateDepth) { return nil, gas, ErrDepth } - // Make sure the readonly is only set if we aren't in readonly yet - // this makes also sure that the readonly flag isn't removed for - // child calls. - if !evm.interpreter.IsReadOnly() { - evm.interpreter.SetReadOnly(true) - defer func() { evm.interpreter.SetReadOnly(false) }() - } var ( to = AccountRef(addr) @@ -331,7 +324,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally // when we're in Homestead this also counts for code storage gas errors. - ret, err = run(evm, contract, input) + ret, err = run(evm, contract, input, true) if err != nil { evm.StateDB.RevertToSnapshot(snapshot) if err != errExecutionReverted { @@ -382,7 +375,7 @@ func (evm *EVM) create(caller ContractRef, code []byte, gas uint64, value *big.I } start := time.Now() - ret, err := run(evm, contract, nil) + ret, err := run(evm, contract, nil, false) // check whether the max code size has been exceeded maxCodeSizeExceeded := evm.ChainConfig().IsEIP158(evm.BlockNumber) && len(ret) > params.MaxCodeSize diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 1e9202424a..0f1b073421 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -48,7 +48,7 @@ type Config struct { type Interpreter interface { // Run loops and evaluates the contract's code with the given input data and returns // the return byte-slice and an error if one occurred. - Run(contract *Contract, input []byte) ([]byte, error) + Run(contract *Contract, input []byte, static bool) ([]byte, error) // CanRun tells if the contract, passed as an argument, can be // run by the current interpreter. This is meant so that the // caller can do something like: @@ -61,10 +61,6 @@ type Interpreter interface { // } // ``` CanRun([]byte) bool - // IsReadOnly reports if the interpreter is in read only mode. - IsReadOnly() bool - // SetReadOnly sets (or unsets) read only mode in the interpreter. - SetReadOnly(bool) } // EVMInterpreter represents an EVM interpreter @@ -125,7 +121,7 @@ func (in *EVMInterpreter) enforceRestrictions(op OpCode, operation operation, st // It's important to note that any errors returned by the interpreter should be // considered a revert-and-consume-all-gas operation except for // errExecutionReverted which means revert-and-keep-gas-left. -func (in *EVMInterpreter) Run(contract *Contract, input []byte) (ret []byte, err error) { +func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { if in.intPool == nil { in.intPool = poolOfIntPools.get() defer func() { @@ -138,6 +134,13 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte) (ret []byte, err in.evm.depth++ defer func() { in.evm.depth-- }() + // Make sure the readOnly is only set if we aren't in readOnly yet. + // This makes also sure that the readOnly flag isn't removed for child calls. + if readOnly && !in.readOnly { + in.readOnly = true + defer func() { in.readOnly = false }() + } + // Reset the previous call's return data. It's unimportant to preserve the old buffer // as every returning call will return new data anyway. in.returnData = nil @@ -263,13 +266,3 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte) (ret []byte, err func (in *EVMInterpreter) CanRun(code []byte) bool { return true } - -// IsReadOnly reports if the interpreter is in read only mode. -func (in *EVMInterpreter) IsReadOnly() bool { - return in.readOnly -} - -// SetReadOnly sets (or unsets) read only mode in the interpreter. -func (in *EVMInterpreter) SetReadOnly(ro bool) { - in.readOnly = ro -} diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index 117c376b81..58b6247245 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -49,7 +49,7 @@ func runTrace(tracer *Tracer) (json.RawMessage, error) { contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000) contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0} - _, err := env.Interpreter().Run(contract, []byte{}) + _, err := env.Interpreter().Run(contract, []byte{}, false) if err != nil { return nil, err } From bcfb7f58b93e6fb5f3da0000672adee80fd6a485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Sep 2018 15:00:54 +0300 Subject: [PATCH 93/96] consensus/clique: only trust snapshot for genesis or les checkpoint --- consensus/clique/clique.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 5472909846..0ff72e55c6 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -388,7 +388,7 @@ func (c *Clique) snapshot(chain consensus.ChainReader, number uint64, hash commo } } // If we're at an checkpoint block, make a snapshot if it's known - if number%c.config.Epoch == 0 { + if number == 0 || (number%c.config.Epoch == 0 && chain.GetHeaderByNumber(number-1) == nil) { checkpoint := chain.GetHeaderByNumber(number) if checkpoint != nil { hash := checkpoint.Hash() From 0e32989a08b8b84e7fe4ae397fe4302e93e34782 Mon Sep 17 00:00:00 2001 From: TColl <38299499+TColl@users.noreply.github.com> Date: Mon, 10 Sep 2018 13:22:34 +0100 Subject: [PATCH 94/96] cmd/utils: typos in {Miner, MinerLegacy}GasPriceFlag (#17588) --- cmd/utils/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f431621baf..0fecae9aa0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -343,12 +343,12 @@ var ( } MinerGasPriceFlag = BigFlag{ Name: "miner.gasprice", - Usage: "Minimal gas price for mining a transactions", + Usage: "Minimum gas price for mining a transaction", Value: eth.DefaultConfig.MinerGasPrice, } MinerLegacyGasPriceFlag = BigFlag{ Name: "gasprice", - Usage: "Minimal gas price for mining a transactions (deprecated, use --miner.gasprice)", + Usage: "Minimum gas price for mining a transaction (deprecated, use --miner.gasprice)", Value: eth.DefaultConfig.MinerGasPrice, } MinerEtherbaseFlag = cli.StringFlag{ From 6dd87483d4ded2a5ab76d19e0acaa5c6b06312a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Tue, 11 Sep 2018 11:39:02 +0200 Subject: [PATCH 95/96] Encryption async api (#17603) * swarm/storage/encryption: async segmentwise encryption/decryption * swarm/storage: adapt hasherstore to encryption API change * swarm/api: adapt RefEncryption for AC to new Encryption API * swarm/storage/encryption: address review comments --- swarm/api/encrypt.go | 22 +-- swarm/storage/encryption/encryption.go | 140 ++++++++++++-------- swarm/storage/encryption/encryption_test.go | 47 ++++--- swarm/storage/hasherstore.go | 100 +++++++------- 4 files changed, 177 insertions(+), 132 deletions(-) diff --git a/swarm/api/encrypt.go b/swarm/api/encrypt.go index 9a2e369149..ffe6c16d2d 100644 --- a/swarm/api/encrypt.go +++ b/swarm/api/encrypt.go @@ -25,27 +25,27 @@ import ( ) type RefEncryption struct { - spanEncryption encryption.Encryption - dataEncryption encryption.Encryption - span []byte + refSize int + span []byte } func NewRefEncryption(refSize int) *RefEncryption { span := make([]byte, 8) binary.LittleEndian.PutUint64(span, uint64(refSize)) return &RefEncryption{ - spanEncryption: encryption.New(0, uint32(refSize/32), sha3.NewKeccak256), - dataEncryption: encryption.New(refSize, 0, sha3.NewKeccak256), - span: span, + refSize: refSize, + span: span, } } func (re *RefEncryption) Encrypt(ref []byte, key []byte) ([]byte, error) { - encryptedSpan, err := re.spanEncryption.Encrypt(re.span, key) + spanEncryption := encryption.New(key, 0, uint32(re.refSize/32), sha3.NewKeccak256) + encryptedSpan, err := spanEncryption.Encrypt(re.span) if err != nil { return nil, err } - encryptedData, err := re.dataEncryption.Encrypt(ref, key) + dataEncryption := encryption.New(key, re.refSize, 0, sha3.NewKeccak256) + encryptedData, err := dataEncryption.Encrypt(ref) if err != nil { return nil, err } @@ -57,7 +57,8 @@ func (re *RefEncryption) Encrypt(ref []byte, key []byte) ([]byte, error) { } func (re *RefEncryption) Decrypt(ref []byte, key []byte) ([]byte, error) { - decryptedSpan, err := re.spanEncryption.Decrypt(ref[:8], key) + spanEncryption := encryption.New(key, 0, uint32(re.refSize/32), sha3.NewKeccak256) + decryptedSpan, err := spanEncryption.Decrypt(ref[:8]) if err != nil { return nil, err } @@ -67,7 +68,8 @@ func (re *RefEncryption) Decrypt(ref []byte, key []byte) ([]byte, error) { return nil, errors.New("invalid span in encrypted reference") } - decryptedRef, err := re.dataEncryption.Decrypt(ref[8:], key) + dataEncryption := encryption.New(key, re.refSize, 0, sha3.NewKeccak256) + decryptedRef, err := dataEncryption.Decrypt(ref[8:]) if err != nil { return nil, err } diff --git a/swarm/storage/encryption/encryption.go b/swarm/storage/encryption/encryption.go index e50f2163db..6fbdab062b 100644 --- a/swarm/storage/encryption/encryption.go +++ b/swarm/storage/encryption/encryption.go @@ -21,6 +21,7 @@ import ( "encoding/binary" "fmt" "hash" + "sync" ) const KeyLength = 32 @@ -28,84 +29,119 @@ const KeyLength = 32 type Key []byte type Encryption interface { - Encrypt(data []byte, key Key) ([]byte, error) - Decrypt(data []byte, key Key) ([]byte, error) + Encrypt(data []byte) ([]byte, error) + Decrypt(data []byte) ([]byte, error) } type encryption struct { - padding int - initCtr uint32 - hashFunc func() hash.Hash + key Key // the encryption key (hashSize bytes long) + keyLen int // length of the key = length of blockcipher block + padding int // encryption will pad the data upto this if > 0 + initCtr uint32 // initial counter used for counter mode blockcipher + hashFunc func() hash.Hash // hasher constructor function } -func New(padding int, initCtr uint32, hashFunc func() hash.Hash) *encryption { +// New constructs a new encryptor/decryptor +func New(key Key, padding int, initCtr uint32, hashFunc func() hash.Hash) *encryption { return &encryption{ + key: key, + keyLen: len(key), padding: padding, initCtr: initCtr, hashFunc: hashFunc, } } -func (e *encryption) Encrypt(data []byte, key Key) ([]byte, error) { +// Encrypt encrypts the data and does padding if specified +func (e *encryption) Encrypt(data []byte) ([]byte, error) { length := len(data) + outLength := length isFixedPadding := e.padding > 0 - if isFixedPadding && length > e.padding { - return nil, fmt.Errorf("Data length longer than padding, data length %v padding %v", length, e.padding) + if isFixedPadding { + if length > e.padding { + return nil, fmt.Errorf("Data length longer than padding, data length %v padding %v", length, e.padding) + } + outLength = e.padding } - - paddedData := data - if isFixedPadding && length < e.padding { - paddedData = make([]byte, e.padding) - copy(paddedData[:length], data) - rand.Read(paddedData[length:]) - } - return e.transform(paddedData, key), nil + out := make([]byte, outLength) + e.transform(data, out) + return out, nil } -func (e *encryption) Decrypt(data []byte, key Key) ([]byte, error) { +// Decrypt decrypts the data, if padding was used caller must know original length and truncate +func (e *encryption) Decrypt(data []byte) ([]byte, error) { length := len(data) if e.padding > 0 && length != e.padding { return nil, fmt.Errorf("Data length different than padding, data length %v padding %v", length, e.padding) } - - return e.transform(data, key), nil + out := make([]byte, length) + e.transform(data, out) + return out, nil } -func (e *encryption) transform(data []byte, key Key) []byte { - dataLength := len(data) - transformedData := make([]byte, dataLength) - hasher := e.hashFunc() - ctr := e.initCtr - hashSize := hasher.Size() - for i := 0; i < dataLength; i += hashSize { - hasher.Write(key) - - ctrBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(ctrBytes, ctr) - - hasher.Write(ctrBytes) - - ctrHash := hasher.Sum(nil) - hasher.Reset() - hasher.Write(ctrHash) - - segmentKey := hasher.Sum(nil) - - hasher.Reset() - - segmentSize := min(hashSize, dataLength-i) - for j := 0; j < segmentSize; j++ { - transformedData[i+j] = data[i+j] ^ segmentKey[j] - } - ctr++ +// +func (e *encryption) transform(in, out []byte) { + inLength := len(in) + wg := sync.WaitGroup{} + wg.Add((inLength-1)/e.keyLen + 1) + for i := 0; i < inLength; i += e.keyLen { + l := min(e.keyLen, inLength-i) + // call transformations per segment (asyncronously) + go func(i int, x, y []byte) { + defer wg.Done() + e.Transcrypt(i, x, y) + }(i/e.keyLen, in[i:i+l], out[i:i+l]) } - return transformedData + // pad the rest if out is longer + pad(out[inLength:]) + wg.Wait() } -func GenerateRandomKey() (Key, error) { - key := make([]byte, KeyLength) - _, err := rand.Read(key) - return key, err +// used for segmentwise transformation +// if in is shorter than out, padding is used +func (e *encryption) Transcrypt(i int, in []byte, out []byte) { + // first hash key with counter (initial counter + i) + hasher := e.hashFunc() + hasher.Write(e.key) + + ctrBytes := make([]byte, 4) + binary.LittleEndian.PutUint32(ctrBytes, uint32(i)+e.initCtr) + hasher.Write(ctrBytes) + + ctrHash := hasher.Sum(nil) + hasher.Reset() + + // second round of hashing for selective disclosure + hasher.Write(ctrHash) + segmentKey := hasher.Sum(nil) + hasher.Reset() + + // XOR bytes uptil length of in (out must be at least as long) + inLength := len(in) + for j := 0; j < inLength; j++ { + out[j] = in[j] ^ segmentKey[j] + } + // insert padding if out is longer + pad(out[inLength:]) +} + +func pad(b []byte) { + l := len(b) + for total := 0; total < l; { + read, _ := rand.Read(b[total:]) + total += read + } +} + +// GenerateRandomKey generates a random key of length l +func GenerateRandomKey(l int) Key { + key := make([]byte, l) + var total int + for total < l { + read, _ := rand.Read(key[total:]) + total += read + } + return key } func min(x, y int) int { diff --git a/swarm/storage/encryption/encryption_test.go b/swarm/storage/encryption/encryption_test.go index 5ea546d6bf..c3abefdcec 100644 --- a/swarm/storage/encryption/encryption_test.go +++ b/swarm/storage/encryption/encryption_test.go @@ -22,34 +22,42 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto/sha3" ) -var expectedTransformedHex = "470c6c67ba1820d5cb4c23ccef22aa2417a323dc97e5f5dced930d74f2932fd178df80ddf129f4f8a4ccec0225c1c2e6765cbbd92bbad8413c50d93d53f7b2fec975d6f29468eccdaf6458f7a3306a7bc207211ea7f9ee5e6951ce6874aef09eb7ff9bed0aa00920b4dcc105e5f1f8dfbdb0564751311d6ceaca1e4e6d988097582638106404c03cfaef8db0e46674d9e8192c1b62d1cd952389cab3a8dee9329fdb059bafd9bae3df3a6b5a9a10961a0333016c99ee3d65cf18ea8ea7b4c386d59dcbf317460d517d5b55504b5992ffb9c3d7c49ebe1fe3ab7b00ad84b1d76d95f8165141260e6980b2ffe8a0000c36dd77ace4c7a781887f831f740d92d8b4848f1b9e237877b988b5a3e23d7b07b2c8eda0ea9fa748ab10bb6bfd7447b66f44e528d24eff31defdecddaf2bb5e6b8e2aa3d2f1a83de44e0ee3789b75dadba1375b7f3a8f7f1eb2d9b78ad1844d425fb76ea6ba45eabe88f1e062d7552d8c7d44c9c66c2753c41892cb675fd1d564b5f4746e76aa4b24ad121a907b8786715483067f46f1cd4a3dad5125f58c3d348db776e99e8a562ebe603e0e98509c72118ee81259b43f1f770dc9220dddbe4d12d2ffabdad76663d1cf30304eacbb43e9bbe0e4f95aae8609bf8a0786e56e0d20d4875d17cec644ca16824424c6c0700e5082ad5288d1a9fec637b75d24c270086a3606b97cc3240314ac123c04ff4d67ef547504f0eeea5e6252f9b5b75e47d32e583c4dee95ac84b6f03fef412e24c09697d6f7bc8408c2118c524f8e277b82658a5869d6d69d7fde469be27ea33fa22b47f3276f60580f2a05b0fbceb67949d2c8e6d8b097ceef6781d962866b23bc71b54597dd05caafbd533ff4e0013f0fc7e202782ba1e6ccd11242a5bc823af9ad0d5bcb8b78d0ce150fc6f8b97c66f9337adaf3878cbb70733044fd14f318d8e389940d07e0593e35f46dc84e7a9a75dadda153bbe5755af0683cfd262ce3797b1bc648a4e5cef5d9aa47d08cf78c6c9dd872096deb856680d5d932abfd5f2024f908ab84d19b762fdcd31209b1d9adf9cbab904bc58f5b00d5a3e0eb2ade52c7c9a678aadedb1b9257263e6145d34ebcc250d0867e2fdc2ed23b4a8e8aa580424308b543b76595f7e09127e8c301d3f6c228e656df2b5e63a89d634db2f87249025ad3b53d2703d6b93c4ab6b1dee4f9b4d2a0b383ed87458e0e1b2a3c04f9a08b885d856ea7ca9abda9a9c8bd28ecce78675b16829493dbeb66963a3665a8386bc4b97d13cfacacfa80b12e36b598d4f09ab7847481a95b35a937bd4f0107598844fc518c4134e8ca55cec024773dcf0a17c71f602406363dcc03912f23cffad9c611e34605fb03fea3d84aac33886efed6c53d455c786231436c01524cd41e2b6c35b339feaa94d7df0b3572db5595c05a1d4a96e1a17e814cea965be81786784af5697b5c7909fd7701d0f5d991f469e20fbc39536239c7a53278c0b1189253d3d978d54b51d33e0c8ac03ca1f08cfd440130944ae9d353b0b7f263a3a27d15290f35c0ad7cf62e8429ebfa10f5471987c53c5d574c9aebb935bdb8339fee9e1030b3f2845c36272642233afba71f81f10322057b51a33b434a4bc45530581c4a636895fc773e20cf6b8a6cef060920310f4cb79f735fc14e758f646b0cf3bf0ce1f1479788f5b73507203d54022c7afd805d46f45e7ede96b2660869939440d7d575fbf372a7f63421728343dfd88c5fa29ea3f15b68fe360f54afa0681093f2c82888d6babf9558159383c1a76264deec2d7fccb20dbd3ab4b3f137fb3dcf253fb6953df2f663d08e8b39909c9dbff98d49377ce0ce03c580b5e1bca6fcf58aceda42b00b1f478cff2995d47c82f2d5a3dc0e9718a0a46d8330353f2039e68d9565d29e77efea91e058ae03140a4b58b297ae664c9f0e7af78f3633f4aeabd95fd367ac5c67eb63fa2bd7035f4adfe3ab8502952d1675b47118b2006b4a5a1b2a3d03aa670862adc5f1ad1b39f0e4a08e2412128e0ef5fa84366b4b50992cc139e7837f8d65f8eae3c8dc2730f2ef1e1585ce95b0fe354c6c853526558a1732171b6017254ba0f0c22273b55caa2ee680791fb34e22bf897a0998156083c83d03310956b1f947ef474c90b2d7e20beae27f5d33b79ad6d9b1b0188e3cf850108f3a02f9386a314b97fb6c946e572e40024da3e4784c523cf2a70e45f6c0f8f35e9b279aecee183ccb30477ee4f5d57c246ab9918495159307935340a92bd46d6519bcc51af4a785a7eb7fa6eae9def89e2411efd2cb2c33f4d605b77af1ae298967fd846453e8b0a55c57706e79d7badd93268d93be27790bbb051496552b094f37bf96843cdf7604dc7f696976bebbe3561c7bd4b3f2843bd07e34a3c3acf0d6c755014a3e9d922dabdbc892511b6af3214958b3531baceab9082c2b4e19ef802b99db2cfa4076ba681efae8a9546582e4cbf07000fb3903727261b93d1bd08809ddc41b2a61b0bfa6b9210b8731394fd9084bc1406bcdd0992410e90f749a087edafb76f6617209855269bb5c89711d2830ea99f2a0e548991ef8dd3e62ced239dff4e9d8c1e834fb57078dc2a4322acc1ea1dbe64064db4cc2a7cdc32884cb7c31aa3c95634356b4d32f1c92fa039e5ed18c1af6605e2f66e5383fa1fcc610a3cc1eef5fe6296ac378f2440d0bd3c77d458af0bfa6b64cedd0301b116efafbbb8f9e88d4bedf56d11cfe0967fc06932e1f232c7daf2c73b58c52daa82dc22b2290147e358348f991f1473c4e63ac943cabc429f5689da8aee6801fd941778966da87fe137b033a0231a90a2ac759efbcb7c52de8e8f27bcc4e5bcb560524d17b0345727f8092e22b6a0e03ceee355085d4fe81568b5b8b84b69870dd333c9b3bdca45db7e43b876a217819928fec31f3ca4c44c4f03a91d578f9ed883bf1247a10eefe42279484af6f70719e193b07a14fc7d93fd6cee9e883a81cc51b53044fa2f1b2525c10e23fb988b31559e1fae4c082d905486d984d2f1e87c40976a571f92e4fdf09e23cad561d9cd3d1ca94e778d08b82852dbe9bab2c7f147c6078504c30fc3fe749fca95ed23791dcf6b935ab0ed08645c481cd2b7969258405f183593617c0d24ebe79bc6c87df65ddc25f933b12a641cc95cf321c1b2edf5db5813a98d99fd1760b5fac19c2b47c2a750e96da49c97276cf31cbd73e1e93ef6990fdf1b08a3e7963dffcf65aa6496727f6be5d9225cfbbadfd6a3a06a454acbb80e761699be97b9caeb71dc3c20a2c253f349ee190b9d8c1ff95751a281ed88b098e4a78fae8249232d4280daf46580ca6c14411f8c4d0e8fa44f8808a5a2dda36d35f79aa77c1c4c615216ec309aa5f3caff3a449b9740baad11e525a5651416a12dd4688897b5ee5e1e8361a87269e9be0789ea7564f64752788f3ac57bda4aa45582d8dabc4d44759639984514d9b151004bf0d9d3140becde41603ffd9d412ec08a1b2f7262862a761d7ce5b91a239d68cdfe7c615b62217422c5a224e652cffdbbe7b41064c9375bbf2c23b63130afedd93987c1f1e4a1623a77d3ef4960fd232afc7d12159c8f1245984e4bd390685668f10da77bf5f6130a5d405a42fb02d0048d63ca48cd4901f3704975b0493714c78de33c1a969993529bac13e310ee098a7b1e8e93b4fcb269773a1521defef98f403f79a8e24e1594b8dc6055223b79b4d7009498c02790a37dd3a9ac7168b11568877ee852395165f87d881355327e57ba48be948b1325824244a552ca80c2ecb79a04a199f0b6a58b455327d998a67ebae414394928161a311de9d3d7f1fc99c0bc35b0a814e7821d2d6958129775c56fdd9b28b6e43a3132a95eb64805f9941b0c5ca2078c37bc7a4d43ddcbc6e8c66de2ef51c714a81913bebeb8f9c6aa14da7e71854870fba9f093aa77abeb23e8d37f3a50135a3f894d1cdf647d3dceb3f16ad4b4f01dffe99d74cd1f18d37735ac69287df0a8b49b2fc3773bf426c881b64a1241c1143e39b6dca59e28c116947aac39585dd81f6fe3832f97d873884738982976f9f29aa6463662cc81c2abbd5eafbb3d9800ba811410a521065d61ad01cfde2c2d98f5fa0ec63e8b01ec6a2d738a1f6cf8c99725079aa51ed3dc7630deeea0b27ee8edbfa19bdb8504cf80a1ca74ea6777fa8786fd76d4ab019c00da35278bf49be5a42aff264875b69bab9e7f3f9299f2ea6a883fc5fd77e34debf414ce8af37a3e9a7de05f33bc547899d25a13389d9048be8b2b7c9b836b401eac9c53ba3d44a14e683a6b378248fdedcbfbc408e5ec83a6c3f769533ae90e339222d80d433ae820fca43d80bca7fbd9d6e5eb760271c74a95a35a853285f7dcfe10182af922508501d269cc53ed38756e8d202b25782ec208a39981ef6c9e9342eb6cee2142471574ab39ebdbbf140f64e305d6f6aa73d49d5bb765347a27ebd0ba1bdcaa8a383526b41f3361faddafec726e185cc223f8465472d7a53ad2d74717960ba5684bc692f773e680d0247f9de1650c046ea56fa18c1c3fb6636327f863f37dca86867c57cc643ff6b7973f97c91716502c33f0dc66176357315f09064c70df7db026413567ac9ec7605acd80606d962f7463359998b92f8a33ccffc80f6e162c1b5a65404e0cc4e37688ded910e6ae5f774e3238a1248b05c53486d149734dd1e38073603df0aa4a4f01fd0c1a0e82b6a513f7d22d1b09ff107923462fbfffbf5fb1f2c043e305917eee1f7f18f62dc604a1d39e5bf675dca67da52f95d312b31be7720efd391bf4794f0555b299325b5dfea9eb00e695861634ad28717cffff266e30c864ba844b0c3fd88bd0cee4d929530c2bb3663c5170a15a66c412ff2fed2d962dd0ff145f19f8085931cbd6bde4c11c9c2debfbdb6748d1a6dabf1404762343d468944a0495026091bc44c69dad971890b7221e1eac2e985097d344a4e2375938aa93806ad1715be8d05f4068ec67ef411e704b01851b7427c4137e6dadedfbb25378ecf1c6b749214f2655cf43fdbb72cb842aaab3074cd792f30f96e874d92f2ef62667d81ccd663dfd969c18dff790b6b7b261f0a03baa85bfd7db72baffe2ddc4cc3985183301a77daa826b1f76f6b6f2ac075c6a2b86609e4d26eb08f3ed6f341d7946966e654ed2c30a629ff81a57014a84105a9ad36a525033f16e3e60d639bf8f89ae6cfdbcdb41e93859354957dcfe9a847757c3cd946d8994cda126f146b77119bbc87c49ce79ad844715cd9bb0c7a4f800fce14c81d175aed59fce0377e62c6e597ab5acf1b3b7100403f371c19dd0f131c4c572e57e3e13743f9bac24a6a177d71f03c5d185bb7e163ec5866dda629340a964bc442d423ad7bc5187f3da68d1498dfe9f1815b31bae11df3585ece230cc3521f7a0b4b3360ddf898984b528afff75f229915f1f2d5c4491c65e2f38d26c0de7dc483860da7bf52859fdbc21946badec0bbf00ba8143b8b7289cfb7096d3405e3183e56f1cbb8c48f25d058530894d0302cdce7c43ea31768b5b610820c97f6e9a8e31a8e9c4624117d213a03ef7d655513cffd8fb5606fc8790692c99828a47382e3e37b9c1317028f5c9d196ff3c2f09435df7614fb37ea20b2371c52b6c4667799922010edeb28001de2a137889db4d3dbf0a406ba90f3631be485ec5e4110b01502f557f15026716030fb6384499adee3cfc44015638e05b206ce3cbc3cbf21b7dd1dcb3b932629e7cd4f4f6b148f37f976803644e5ce792583daf39608a3cd02ff2f47e9bc6" +var expectedTransformedHex = "352187af3a843decc63ceca6cb01ea39dbcf77caf0a8f705f5c30d557044ceec9392b94a79376f1e5c10cd0c0f2a98e5353bf22b3ea4fdac6677ee553dec192e3db64e179d0474e96088fb4abd2babd67de123fb398bdf84d818f7bda2c1ab60b3ea0e0569ae54aa969658eb4844e6960d2ff44d7c087ee3aaffa1c0ee5df7e50b615f7ad90190f022934ad5300c7d1809bfe71a11cc04cece5274eb97a5f20350630522c1dbb7cebaf4f97f84e03f5cfd88f2b48880b25d12f4d5e75c150f704ef6b46c72e07db2b705ac3644569dccd22fd8f964f6ef787fda63c46759af334e6f665f70eac775a7017acea49f3c7696151cb1b9434fa4ac27fb803921ffb5ec58dafa168098d7d5b97e384be3384cf5bc235c3d887fef89fe76c0065f9b8d6ad837b442340d9e797b46ef5709ea3358bc415df11e4830de986ef0f1c418ffdcc80e9a3cda9bea0ab5676c0d4240465c43ba527e3b4ea50b4f6255b510e5d25774a75449b0bd71e56c537ade4fcf0f4d63c99ae1dbb5a844971e2c19941b8facfcfc8ee3056e7cb3c7114c5357e845b52f7103cb6e00d2308c37b12baa5b769e1cc7b00fc06f2d16e70cc27a82cb9c1a4e40cb0d43907f73df2c9db44f1b51a6b0bc6d09f77ac3be14041fae3f9df2da42df43ae110904f9ecee278030185254d7c6e918a5512024d047f77a992088cb3190a6587aa54d0c7231c1cd2e455e0d4c07f74bece68e29cd8ba0190c0bcfb26d24634af5d91a81ef5d4dd3d614836ce942ddbf7bb1399317f4c03faa675f325f18324bf9433844bfe5c4cc04130c8d5c329562b7cd66e72f7355de8f5375a72202971613c32bd7f3fcdcd51080758cd1d0a46dbe8f0374381dbc359f5864250c63dde8131cbd7c98ae2b0147d6ea4bf65d1443d511b18e6d608bbb46ac036353b4c51df306a10a6f6939c38629a5c18aaf89cac04bd3ad5156e6b92011c88341cb08551bab0a89e6a46538f5af33b86121dba17e3a434c273f385cd2e8cb90bdd32747d8425d929ccbd9b0815c73325988855549a8489dfd047daf777aaa3099e54cf997175a5d9e1edfe363e3b68c70e02f6bf4fcde6a0f3f7d0e7e98bde1a72ae8b6cd27b32990680cc4a04fc467f41c5adcaddabfc71928a3f6872c360c1d765260690dd28b269864c8e380d9c92ef6b89b0094c8f9bb22608b4156381b19b920e9583c9616ce5693b4d2a6c689f02e6a91584a8e501e107403d2689dd0045269dd9946c0e969fb656a3b39d84a798831f5f9290f163eb2f97d3ae25071324e95e2256d9c1e56eb83c26397855323edc202d56ad05894333b7f0ed3c1e4734782eb8bd5477242fd80d7a89b12866f85cfae476322f032465d6b1253993033fccd4723530630ab97a1566460af9c90c9da843c229406e65f3fa578bd6bf04dee9b6153807ddadb8ceefc5c601a8ab26023c67b1ab1e8e0f29ce94c78c308005a781853e7a2e0e51738939a657c987b5e611f32f47b5ff461c52e63e0ea390515a8e1f5393dae54ea526934b5f310b76e3fa050e40718cb4c8a20e58946d6ee1879f08c52764422fe542b3240e75eccb7aa75b1f8a651e37a3bc56b0932cdae0e985948468db1f98eb4b77b82081ea25d8a762db00f7898864984bd80e2f3f35f236bf57291dec28f550769943bcfb6f884b7687589b673642ef7fe5d7d5a87d3eca5017f83ccb9a3310520474479464cb3f433440e7e2f1e28c0aef700a45848573409e7ab66e0cfd4fe5d2147ace81bc65fd8891f6245cd69246bbf5c27830e5ab882dd1d02aba34ff6ca9af88df00fd602892f02fedbdc65dedec203faf3f8ff4a97314e0ddb58b9ab756a61a562597f4088b445fcc3b28a708ca7b1485dcd791b779fbf2b3ef1ec5c6205f595fbe45a02105034147e5a146089c200a49dae33ae051a08ea5f974a21540aaeffa7f9d9e3d35478016fb27b871036eb27217a5b834b461f535752fb5f1c8dded3ae14ce3a2ef6639e2fe41939e3509e46e347a95d50b2080f1ba42c804b290ddc912c952d1cec3f2661369f738feacc0dbf1ea27429c644e45f9e26f30c341acd34c7519b2a1663e334621691e810767e9918c2c547b2e23cce915f97d26aac8d0d2fcd3edb7986ad4e2b8a852edebad534cb6c0e9f0797d3563e5409d7e068e48356c67ce519246cd9c560e881453df97cbba562018811e6cf8c327f399d1d1253ab47a19f4a0ccc7c6d86a9603e0551da310ea595d71305c4aad96819120a92cdbaf1f77ec8df9cc7c838c0d4de1e8692dd81da38268d1d71324bcffdafbe5122e4b81828e021e936d83ae8021eac592aa52cd296b5ce392c7173d622f8e07d18f59bb1b08ba15211af6703463b09b593af3c37735296816d9f2e7a369354a5374ea3955e14ca8ac56d5bfe4aef7a21bd825d6ae85530bee5d2aaaa4914981b3dfdb2e92ec2a27c83d74b59e84ff5c056f7d8945745f2efc3dcf28f288c6cd8383700fb2312f7001f24dd40015e436ae23e052fe9070ea9535b9c989898a9bda3d5382cf10e432fae6ccf0c825b3e6436edd3a9f8846e5606f8563931b5f29ba407c5236e5730225dda211a8504ec1817bc935e1fd9a532b648c502df302ed2063aed008fd5676131ac9e95998e9447b02bd29d77e38fcfd2959f2de929b31970335eb2a74348cc6918bc35b9bf749eab0fe304c946cd9e1ca284e6853c42646e60b6b39e0d3fb3c260abfc5c1b4ca3c3770f344118ca7c7f5c1ad1f123f8f369cd60afc3cdb3e9e81968c5c9fa7c8b014ffe0508dd4f0a2a976d5d1ca8fc9ad7a237d92cfe7b41413d934d6e142824b252699397e48e4bac4e91ebc10602720684bd0863773c548f9a2f9724245e47b129ecf65afd7252aac48c8a8d6fd3d888af592a01fb02dc71ed7538a700d3d16243e4621e0fcf9f8ed2b4e11c9fa9a95338bb1dac74a7d9bc4eb8cbf900b634a2a56469c00f5994e4f0934bdb947640e6d67e47d0b621aacd632bfd3c800bd7d93bd329f494a90e06ed51535831bd6e07ac1b4b11434ef3918fa9511813a002913f33f836454798b8d1787fea9a4c4743ba091ed192ed92f4d33e43a226bf9503e1a83a16dd340b3cbbf38af6db0d99201da8de529b4225f3d2fa2aad6621afc6c79ef3537720591edfc681ae6d00ede53ed724fc71b23b90d2e9b7158aaee98d626a4fe029107df2cb5f90147e07ebe423b1519d848af18af365c71bfd0665db46be493bbe99b79a188de0cf3594aef2299f0324075bdce9eb0b87bc29d62401ba4fd6ae48b1ba33261b5b845279becf38ee03e3dc5c45303321c5fac96fd02a3ad8c9e3b02127b320501333c9e6360440d1ad5e64a6239501502dde1a49c9abe33b66098458eee3d611bb06ffcd234a1b9aef4af5021cd61f0de6789f822ee116b5078aae8c129e8391d8987500d322b58edd1595dc570b57341f2df221b94a96ab7fbcf32a8ca9684196455694024623d7ed49f7d66e8dd453c0bae50e0d8b34377b22d0ece059e2c385dfc70b9089fcd27577c51f4d870b5738ee2b68c361a67809c105c7848b68860a829f29930857a9f9d40b14fd2384ac43bafdf43c0661103794c4bd07d1cfdd4681b6aeaefad53d4c1473359bcc5a83b09189352e5bb9a7498dd0effb89c35aad26954551f8b0621374b449bf515630bd3974dca982279733470fdd059aa9c3df403d8f22b38c4709c82d8f12b888e22990350490e16179caf406293cc9e65f116bafcbe96af132f679877061107a2f690a82a8cb46eea57a90abd23798c5937c6fe6b17be3f9bfa01ce117d2c268181b9095bf49f395fea07ca03838de0588c5e2db633e836d64488c1421e653ea52d810d096048c092d0da6e02fa6613890219f51a76148c8588c2487b171a28f17b7a299204874af0131725d793481333be5f08e86ca837a226850b0c1060891603bfecf9e55cddd22c0dbb28d495342d9cc3de8409f72e52a0115141cffe755c74f061c1a770428ccb0ae59536ee6fc074fbfc6cacb51a549d327527e20f8407477e60355863f1153f9ce95641198663c968874e7fdb29407bd771d94fdda8180cbb0358f5874738db705924b8cbe0cd5e1484aeb64542fe8f38667b7c34baf818c63b1e18440e9fba575254d063fd49f24ef26432f4eb323f3836972dca87473e3e9bb26dc3be236c3aae6bc8a6da567442309da0e8450e242fc9db836e2964f2c76a3b80a2c677979882dda7d7ebf62c93664018bcf4ec431fe6b403d49b3b36618b9c07c2d0d4569cb8d52223903debc72ec113955b206c34f1ae5300990ccfc0180f47d91afdb542b6312d12aeff7e19c645dc0b9fe6e3288e9539f6d5870f99882df187bfa6d24d179dfd1dac22212c8b5339f7171a3efc15b760fed8f68538bc5cbd845c2d1ab41f3a6c692820653eaef7930c02fbe6061d93805d73decdbb945572a7c44ed0241982a6e4d2d730898f82b3d9877cb7bca41cc6dcee67aa0c3d6db76f0b0a708ace0031113e48429de5d886c10e9200f68f32263a2fbf44a5992c2459fda7b8796ba796e3a0804fc25992ed2c9a5fe0580a6b809200ecde6caa0364b58be11564dcb9a616766dd7906db5636ee708b0204f38d309466d8d4a162965dd727e29f5a6c133e9b4ed5bafe803e479f9b2a7640c942c4a40b14ac7dc9828546052761a070f6404008f1ec3605836339c3da95a00b4fd81b2cabf88b51d2087d5b83e8c5b69bf96d8c72cbd278dad3bbb42b404b436f84ad688a22948adf60a81090f1e904291503c16e9f54b05fc76c881a5f95f0e732949e95d3f1bae2d3652a14fe0dda2d68879604657171856ef72637def2a96ac47d7b3fe86eb3198f5e0e626f06be86232305f2ae79ffcd2725e48208f9d8d63523f81915acc957563ab627cd6bc68c2a37d59fb0ed77a90aa9d085d6914a8ebada22a2c2d471b5163aeddd799d90fbb10ed6851ace2c4af504b7d572686700a59d6db46d5e42bb83f8e0c0ffe1dfa6582cc0b34c921ff6e85e83188d24906d5c08bb90069639e713051b3102b53e6f703e8210017878add5df68e6f2b108de279c5490e9eef5590185c4a1c744d4e00d244e1245a8805bd30407b1bc488db44870ccfd75a8af104df78efa2fb7ba31f048a263efdb3b63271fff4922bece9a71187108f65744a24f4947dc556b7440cb4fa45d296bb7f724588d1f245125b21ea063500029bd49650237f53899daf1312809552c81c5827341263cc807a29fe84746170cdfa1ff3838399a5645319bcaff674bb70efccdd88b3d3bb2f2d98111413585dc5d5bd5168f43b3f55e58972a5b2b9b3733febf02f931bd436648cb617c3794841aab961fe41277ab07812e1d3bc4ff6f4350a3e615bfba08c3b9480ef57904d3a16f7e916345202e3f93d11f7a7305170cb8c4eb9ac88ace8bbd1f377bdd5855d3162d6723d4435e84ce529b8f276a8927915ac759a0d04e5ca4a9d3da6291f0333b475df527e99fe38f7a4082662e8125936640c26dd1d17cf284ce6e2b17777a05aa0574f7793a6a062cc6f7263f7ab126b4528a17becfdec49ac0f7d8705aa1704af97fb861faa8a466161b2b5c08a5bacc79fe8500b913d65c8d3c52d1fd52d2ab2c9f52196e712455619c1cd3e0f391b274487944240e2ed8858dd0823c801094310024ae3fe4dd1cf5a2b6487b42cc5937bbafb193ee331d87e378258963d49b9da90899bbb4b88e79f78e866b0213f4719f67da7bcc2fce073c01e87c62ea3cdbcd589cfc41281f2f4c757c742d6d1e" var hashFunc = sha3.NewKeccak256 +var testKey Key + +func init() { + var err error + testKey, err = hexutil.Decode("0x8abf1502f557f15026716030fb6384792583daf39608a3cd02ff2f47e9bc6e49") + if err != nil { + panic(err.Error()) + } +} func TestEncryptDataLongerThanPadding(t *testing.T) { - enc := New(4095, uint32(0), hashFunc) + enc := New(testKey, 4095, uint32(0), hashFunc) data := make([]byte, 4096) - key := make([]byte, 32) expectedError := "Data length longer than padding, data length 4096 padding 4095" - _, err := enc.Encrypt(data, key) + _, err := enc.Encrypt(data) if err == nil || err.Error() != expectedError { t.Fatalf("Expected error \"%v\" got \"%v\"", expectedError, err) } } func TestEncryptDataZeroPadding(t *testing.T) { - enc := New(0, uint32(0), hashFunc) + enc := New(testKey, 0, uint32(0), hashFunc) data := make([]byte, 2048) - key := make([]byte, 32) - encrypted, err := enc.Encrypt(data, key) + encrypted, err := enc.Encrypt(data) if err != nil { t.Fatalf("Expected no error got %v", err) } @@ -59,12 +67,11 @@ func TestEncryptDataZeroPadding(t *testing.T) { } func TestEncryptDataLengthEqualsPadding(t *testing.T) { - enc := New(4096, uint32(0), hashFunc) + enc := New(testKey, 4096, uint32(0), hashFunc) data := make([]byte, 4096) - key := make([]byte, 32) - encrypted, err := enc.Encrypt(data, key) + encrypted, err := enc.Encrypt(data) if err != nil { t.Fatalf("Expected no error got %v", err) } @@ -77,12 +84,11 @@ func TestEncryptDataLengthEqualsPadding(t *testing.T) { } func TestEncryptDataLengthSmallerThanPadding(t *testing.T) { - enc := New(4096, uint32(0), hashFunc) + enc := New(testKey, 4096, uint32(0), hashFunc) data := make([]byte, 4080) - key := make([]byte, 32) - encrypted, err := enc.Encrypt(data, key) + encrypted, err := enc.Encrypt(data) if err != nil { t.Fatalf("Expected no error got %v", err) } @@ -96,14 +102,13 @@ func TestEncryptDataCounterNonZero(t *testing.T) { } func TestDecryptDataLengthNotEqualsPadding(t *testing.T) { - enc := New(4096, uint32(0), hashFunc) + enc := New(testKey, 4096, uint32(0), hashFunc) data := make([]byte, 4097) - key := make([]byte, 32) expectedError := "Data length different than padding, data length 4097 padding 4096" - _, err := enc.Decrypt(data, key) + _, err := enc.Decrypt(data) if err == nil || err.Error() != expectedError { t.Fatalf("Expected error \"%v\" got \"%v\"", expectedError, err) } @@ -117,20 +122,18 @@ func TestEncryptDecryptIsIdentity(t *testing.T) { } func testEncryptDecryptIsIdentity(t *testing.T, padding int, initCtr uint32, dataLength int, keyLength int) { - enc := New(padding, initCtr, hashFunc) + key := GenerateRandomKey(keyLength) + enc := New(key, padding, initCtr, hashFunc) data := make([]byte, dataLength) rand.Read(data) - key := make([]byte, keyLength) - rand.Read(key) - - encrypted, err := enc.Encrypt(data, key) + encrypted, err := enc.Encrypt(data) if err != nil { t.Fatalf("Expected no error got %v", err) } - decrypted, err := enc.Decrypt(encrypted, key) + decrypted, err := enc.Decrypt(encrypted) if err != nil { t.Fatalf("Expected no error got %v", err) } diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index bc23077c18..766207eae0 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -26,49 +26,34 @@ import ( "github.com/ethereum/go-ethereum/swarm/storage/encryption" ) -type chunkEncryption struct { - spanEncryption encryption.Encryption - dataEncryption encryption.Encryption -} - type hasherStore struct { - store ChunkStore - hashFunc SwarmHasher - chunkEncryption *chunkEncryption - hashSize int // content hash size - refSize int64 // reference size (content hash + possibly encryption key) - wg *sync.WaitGroup - closed chan struct{} -} - -func newChunkEncryption(chunkSize, refSize int64) *chunkEncryption { - return &chunkEncryption{ - spanEncryption: encryption.New(0, uint32(chunkSize/refSize), sha3.NewKeccak256), - dataEncryption: encryption.New(int(chunkSize), 0, sha3.NewKeccak256), - } + store ChunkStore + toEncrypt bool + hashFunc SwarmHasher + hashSize int // content hash size + refSize int64 // reference size (content hash + possibly encryption key) + wg *sync.WaitGroup + closed chan struct{} } // NewHasherStore creates a hasherStore object, which implements Putter and Getter interfaces. // With the HasherStore you can put and get chunk data (which is just []byte) into a ChunkStore // and the hasherStore will take core of encryption/decryption of data if necessary func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore { - var chunkEncryption *chunkEncryption - hashSize := hashFunc().Size() refSize := int64(hashSize) if toEncrypt { refSize += encryption.KeyLength - chunkEncryption = newChunkEncryption(chunk.DefaultSize, refSize) } return &hasherStore{ - store: chunkStore, - hashFunc: hashFunc, - chunkEncryption: chunkEncryption, - hashSize: hashSize, - refSize: refSize, - wg: &sync.WaitGroup{}, - closed: make(chan struct{}), + store: chunkStore, + toEncrypt: toEncrypt, + hashFunc: hashFunc, + hashSize: hashSize, + refSize: refSize, + wg: &sync.WaitGroup{}, + closed: make(chan struct{}), } } @@ -79,7 +64,7 @@ func (h *hasherStore) Put(ctx context.Context, chunkData ChunkData) (Reference, c := chunkData size := chunkData.Size() var encryptionKey encryption.Key - if h.chunkEncryption != nil { + if h.toEncrypt { var err error c, encryptionKey, err = h.encryptChunkData(chunkData) if err != nil { @@ -155,23 +140,14 @@ func (h *hasherStore) encryptChunkData(chunkData ChunkData) (ChunkData, encrypti return nil, nil, fmt.Errorf("Invalid ChunkData, min length 8 got %v", len(chunkData)) } - encryptionKey, err := encryption.GenerateRandomKey() - if err != nil { - return nil, nil, err - } - - encryptedSpan, err := h.chunkEncryption.spanEncryption.Encrypt(chunkData[:8], encryptionKey) - if err != nil { - return nil, nil, err - } - encryptedData, err := h.chunkEncryption.dataEncryption.Encrypt(chunkData[8:], encryptionKey) + key, encryptedSpan, encryptedData, err := h.encrypt(chunkData) if err != nil { return nil, nil, err } c := make(ChunkData, len(encryptedSpan)+len(encryptedData)) copy(c[:8], encryptedSpan) copy(c[8:], encryptedData) - return c, encryptionKey, nil + return c, key, nil } func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryption.Key) (ChunkData, error) { @@ -179,12 +155,7 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp return nil, fmt.Errorf("Invalid ChunkData, min length 8 got %v", len(chunkData)) } - decryptedSpan, err := h.chunkEncryption.spanEncryption.Decrypt(chunkData[:8], encryptionKey) - if err != nil { - return nil, err - } - - decryptedData, err := h.chunkEncryption.dataEncryption.Decrypt(chunkData[8:], encryptionKey) + decryptedSpan, decryptedData, err := h.decrypt(chunkData, encryptionKey) if err != nil { return nil, err } @@ -201,13 +172,46 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp copy(c[:8], decryptedSpan) copy(c[8:], decryptedData[:length]) - return c[:length+8], nil + return c, nil } func (h *hasherStore) RefSize() int64 { return h.refSize } +func (h *hasherStore) encrypt(chunkData ChunkData) (encryption.Key, []byte, []byte, error) { + key := encryption.GenerateRandomKey(encryption.KeyLength) + encryptedSpan, err := h.newSpanEncryption(key).Encrypt(chunkData[:8]) + if err != nil { + return nil, nil, nil, err + } + encryptedData, err := h.newDataEncryption(key).Encrypt(chunkData[8:]) + if err != nil { + return nil, nil, nil, err + } + return key, encryptedSpan, encryptedData, nil +} + +func (h *hasherStore) decrypt(chunkData ChunkData, key encryption.Key) ([]byte, []byte, error) { + encryptedSpan, err := h.newSpanEncryption(key).Encrypt(chunkData[:8]) + if err != nil { + return nil, nil, err + } + encryptedData, err := h.newDataEncryption(key).Encrypt(chunkData[8:]) + if err != nil { + return nil, nil, err + } + return encryptedSpan, encryptedData, nil +} + +func (h *hasherStore) newSpanEncryption(key encryption.Key) encryption.Encryption { + return encryption.New(key, 0, uint32(chunk.DefaultSize/h.refSize), sha3.NewKeccak256) +} + +func (h *hasherStore) newDataEncryption(key encryption.Key) encryption.Encryption { + return encryption.New(key, int(chunk.DefaultSize), 0, sha3.NewKeccak256) +} + func (h *hasherStore) storeChunk(ctx context.Context, chunk *Chunk) { h.wg.Add(1) go func() { From 2d98099c25f85a531f0acf28d89ca710f4569c6c Mon Sep 17 00:00:00 2001 From: chenyufeng Date: Tue, 11 Sep 2018 23:05:28 +0800 Subject: [PATCH 96/96] rlp: fix comment typo (#17640) --- rlp/typecache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rlp/typecache.go b/rlp/typecache.go index 3df799e1ec..8c2dd518e2 100644 --- a/rlp/typecache.go +++ b/rlp/typecache.go @@ -76,7 +76,7 @@ func cachedTypeInfo1(typ reflect.Type, tags tags) (*typeinfo, error) { // another goroutine got the write lock first return info, nil } - // put a dummmy value into the cache before generating. + // put a dummy value into the cache before generating. // if the generator tries to lookup itself, it will get // the dummy value and won't call itself recursively. typeCache[key] = new(typeinfo)