diff --git a/.travis.yml b/.travis.yml index a972668c95..8a7f65e3d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ matrix: - os: linux dist: trusty sudo: required - go: 1.7.6 + go: 1.7.x script: - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse - sudo modprobe fuse @@ -18,7 +18,7 @@ matrix: - os: linux dist: trusty sudo: required - go: 1.8.3 + go: 1.8.x script: - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse - sudo modprobe fuse @@ -31,7 +31,7 @@ matrix: - os: linux dist: trusty sudo: required - go: 1.9.0 + go: 1.9.x script: - sudo -E apt-get -yq --no-install-suggests --no-install-recommends --force-yes install fuse - sudo modprobe fuse @@ -41,7 +41,7 @@ matrix: - go run build/ci.go test -coverage -misspell - os: osx - go: 1.9.0 + go: 1.9.x sudo: required script: - brew update @@ -54,7 +54,7 @@ matrix: - os: linux dist: trusty sudo: required - go: 1.9.0 + go: 1.9.x env: - ubuntu-ppa - azure-linux @@ -93,7 +93,7 @@ matrix: sudo: required services: - docker - go: 1.9.0 + go: 1.9.x env: - azure-linux-mips script: @@ -150,7 +150,7 @@ matrix: # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads - os: osx - go: 1.9.0 + go: 1.9.x env: - azure-osx - azure-ios @@ -176,7 +176,7 @@ matrix: - os: linux dist: trusty sudo: required - go: 1.9.0 + go: 1.9.x env: - azure-purge script: diff --git a/VERSION b/VERSION index bd8bf882d0..943f9cbc4e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.7.0 +1.7.1 diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 88fb3331e8..d6d16eb3f9 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -22,6 +22,7 @@ import ( "fmt" "math/big" "sync" + "time" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -284,6 +285,22 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa return nil } +// JumpTimeInSeconds adds skip seconds to the clock +func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { + b.mu.Lock() + defer b.mu.Unlock() + blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), b.database, 1, func(number int, block *core.BlockGen) { + for _, tx := range b.pendingBlock.Transactions() { + block.AddTx(tx) + } + block.OffsetTime(int64(adjustment.Seconds())) + }) + b.pendingBlock = blocks[0] + b.pendingState, _ = state.New(b.pendingBlock.Root(), state.NewDatabase(b.database)) + + return nil +} + // callmsg implements core.Message to allow passing it as a transaction simulator. type callmsg struct { ethereum.CallMsg diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 73e95e02a1..f587580884 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -122,7 +122,7 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La } // For Go bindings pass the code through goimports to clean it up and double check if lang == LangGo { - code, err := imports.Process("", buffer.Bytes(), nil) + code, err := imports.Process(".", buffer.Bytes(), nil) if err != nil { return "", fmt.Errorf("%v\n%s", err, buffer) } diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 8ea3eca415..43ed53b922 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -459,7 +459,7 @@ func TestBindings(t *testing.T) { } // Skip the test if the go-ethereum sources are symlinked (https://github.com/golang/go/issues/14845) linkTestCode := fmt.Sprintf("package linktest\nfunc CheckSymlinks(){\nfmt.Println(backends.NewSimulatedBackend(nil))\n}") - linkTestDeps, err := imports.Process("", []byte(linkTestCode), nil) + linkTestDeps, err := imports.Process(os.TempDir(), []byte(linkTestCode), nil) if err != nil { t.Fatalf("failed check for goimports symlink bug: %v", err) } diff --git a/cmd/puppeth/module_ethstats.go b/cmd/puppeth/module_ethstats.go index 5d3fa5fc03..6ce662f65f 100644 --- a/cmd/puppeth/module_ethstats.go +++ b/cmd/puppeth/module_ethstats.go @@ -42,7 +42,7 @@ RUN \ WORKDIR /eth-netstats EXPOSE 3000 -RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: [{{.Banned}}]};' > lib/utils/config.js +RUN echo 'module.exports = {trusted: [{{.Trusted}}], banned: [{{.Banned}}], reserved: ["yournode"]};' > lib/utils/config.js CMD ["npm", "start"] ` diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 17c258c6c8..23b10c2d74 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -164,7 +164,7 @@ func ImportChain(chain *core.BlockChain, fn string) error { func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool { for _, b := range bs { - if !chain.HasBlock(b.Hash()) { + if !chain.HasBlock(b.Hash(), b.NumberU64()) { return false } } diff --git a/common/types.go b/common/types.go index eaf8352fbc..d31bbf741b 100644 --- a/common/types.go +++ b/common/types.go @@ -88,7 +88,7 @@ func (h Hash) MarshalText() ([]byte, error) { return hexutil.Bytes(h[:]).MarshalText() } -// Sets the hash to the value of b. If b is larger than len(h) it will panic +// Sets the hash to the value of b. If b is larger than len(h), 'b' will be cropped (from the left). func (h *Hash) SetBytes(b []byte) { if len(b) > len(h) { b = b[len(b)-HashLength:] @@ -97,7 +97,7 @@ func (h *Hash) SetBytes(b []byte) { copy(h[HashLength-len(b):], b) } -// Set string `s` to h. If s is larger than len(h) it will panic +// Set string `s` to h. If s is larger than len(h) s will be cropped (from left) to fit. func (h *Hash) SetString(s string) { h.SetBytes([]byte(s)) } // Sets h to other diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index b714204458..6a19d449f3 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -36,9 +36,9 @@ import ( // Ethash proof-of-work protocol constants. var ( - frontierBlockReward *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block - metropolisBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Metropolis - maxUncles = 2 // Maximum number of uncles allowed in a single block + frontierBlockReward *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block + byzantiumBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium + maxUncles = 2 // Maximum number of uncles allowed in a single block ) // Various error messages to mark blocks invalid. These should be private to @@ -290,8 +290,8 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent * func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int { next := new(big.Int).Add(parent.Number, big1) switch { - case config.IsMetropolis(next): - return calcDifficultyMetropolis(time, parent) + case config.IsByzantium(next): + return calcDifficultyByzantium(time, parent) case config.IsHomestead(next): return calcDifficultyHomestead(time, parent) default: @@ -310,10 +310,10 @@ var ( big2999999 = big.NewInt(2999999) ) -// calcDifficultyMetropolis is the difficulty adjustment algorithm. It returns +// calcDifficultyByzantium is the difficulty adjustment algorithm. It returns // the difficulty that a new block should have when created at time given the -// parent block's time and difficulty. The calculation uses the Metropolis rules. -func calcDifficultyMetropolis(time uint64, parent *types.Header) *big.Int { +// parent block's time and difficulty. The calculation uses the Byzantium rules. +func calcDifficultyByzantium(time uint64, parent *types.Header) *big.Int { // https://github.com/ethereum/EIPs/issues/100. // algorithm: // diff = (parent_diff + @@ -530,8 +530,8 @@ var ( func AccumulateRewards(config *params.ChainConfig, state *state.StateDB, header *types.Header, uncles []*types.Header) { // Select the correct block reward based on chain progression blockReward := frontierBlockReward - if config.IsMetropolis(header.Number) { - blockReward = metropolisBlockReward + if config.IsByzantium(header.Number) { + blockReward = byzantiumBlockReward } // Accumulate the rewards for the miner and any included uncles reward := new(big.Int).Set(blockReward) diff --git a/containers/docker/master-alpine/Dockerfile b/containers/docker/master-alpine/Dockerfile index b2467f1a78..44402361d7 100644 --- a/containers/docker/master-alpine/Dockerfile +++ b/containers/docker/master-alpine/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.5 RUN \ apk add --update go git make gcc musl-dev linux-headers ca-certificates && \ - git clone --depth 1 --branch release/1.6 https://github.com/ethereum/go-ethereum && \ + git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \ (cd go-ethereum && make geth) && \ cp go-ethereum/build/bin/geth /geth && \ apk del go git make gcc musl-dev linux-headers && \ diff --git a/containers/docker/master-ubuntu/Dockerfile b/containers/docker/master-ubuntu/Dockerfile index 877ae94e99..cc4fad10c4 100644 --- a/containers/docker/master-ubuntu/Dockerfile +++ b/containers/docker/master-ubuntu/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu:xenial RUN \ apt-get update && apt-get upgrade -q -y && \ apt-get install -y --no-install-recommends golang git make gcc libc-dev ca-certificates && \ - git clone --depth 1 --branch release/1.5 https://github.com/ethereum/go-ethereum && \ + git clone --depth 1 --branch release/1.7 https://github.com/ethereum/go-ethereum && \ (cd go-ethereum && make geth) && \ cp go-ethereum/build/bin/geth /geth && \ apt-get remove -y golang git make gcc libc-dev && apt autoremove -y && apt-get clean && \ diff --git a/contracts/chequebook/cheque_test.go b/contracts/chequebook/cheque_test.go index 5f6a54a1c1..b7555d0815 100644 --- a/contracts/chequebook/cheque_test.go +++ b/contracts/chequebook/cheque_test.go @@ -232,8 +232,8 @@ func TestDeposit(t *testing.T) { balance := new(big.Int).SetUint64(42) chbook.Deposit(balance) backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } amount := common.Big1 @@ -243,8 +243,8 @@ func TestDeposit(t *testing.T) { } backend.Commit() exp := new(big.Int).SetUint64(41) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } // autodeposit on each issue @@ -259,8 +259,8 @@ func TestDeposit(t *testing.T) { t.Fatalf("expected no error, got %v", err) } backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } // autodeposit off @@ -277,11 +277,11 @@ func TestDeposit(t *testing.T) { backend.Commit() exp = new(big.Int).SetUint64(40) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } - // autodeposit every 10ms if new cheque issued + // autodeposit every 30ms if new cheque issued interval := 30 * time.Millisecond chbook.AutoDeposit(interval, common.Big1, balance) _, err = chbook.Issue(addr1, amount) @@ -296,14 +296,14 @@ func TestDeposit(t *testing.T) { backend.Commit() exp = new(big.Int).SetUint64(38) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } time.Sleep(3 * interval) backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } exp = new(big.Int).SetUint64(40) @@ -319,8 +319,8 @@ func TestDeposit(t *testing.T) { } time.Sleep(3 * interval) backend.Commit() - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } _, err = chbook.Issue(addr1, amount) @@ -330,8 +330,8 @@ func TestDeposit(t *testing.T) { time.Sleep(1 * interval) backend.Commit() - if chbook.balance.Cmp(balance) != 0 { - t.Fatalf("expected balance %v, got %v", balance, chbook.balance) + if chbook.Balance().Cmp(balance) != 0 { + t.Fatalf("expected balance %v, got %v", balance, chbook.Balance()) } chbook.AutoDeposit(1*interval, common.Big0, balance) @@ -352,8 +352,8 @@ func TestDeposit(t *testing.T) { backend.Commit() exp = new(big.Int).SetUint64(39) - if chbook.balance.Cmp(exp) != 0 { - t.Fatalf("expected balance %v, got %v", exp, chbook.balance) + if chbook.Balance().Cmp(exp) != 0 { + t.Fatalf("expected balance %v, got %v", exp, chbook.Balance()) } } diff --git a/core/asm/lexer.go b/core/asm/lexer.go index d784e5d504..a34b2cbd8f 100644 --- a/core/asm/lexer.go +++ b/core/asm/lexer.go @@ -145,7 +145,7 @@ func (l *lexer) ignore() { // Accepts checks whether the given input matches the next rune func (l *lexer) accept(valid string) bool { - if strings.IndexRune(valid, l.next()) >= 0 { + if strings.ContainsRune(valid, l.next()) { return true } @@ -157,7 +157,7 @@ func (l *lexer) accept(valid string) bool { // acceptRun will continue to advance the seeker until valid // can no longer be met. func (l *lexer) acceptRun(valid string) { - for strings.IndexRune(valid, l.next()) >= 0 { + for strings.ContainsRune(valid, l.next()) { } l.backup() } @@ -166,7 +166,7 @@ func (l *lexer) acceptRun(valid string) { // to advance the seeker until the rune has been found. func (l *lexer) acceptRunUntil(until rune) bool { // Continues running until a rune is found - for i := l.next(); strings.IndexRune(string(until), i) == -1; i = l.next() { + for i := l.next(); !strings.ContainsRune(string(until), i); i = l.next() { if i == 0 { return false } diff --git a/core/blockchain.go b/core/blockchain.go index d74b3520b1..235c1f3da2 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -23,7 +23,6 @@ import ( "io" "math/big" mrand "math/rand" - "runtime" "sync" "sync/atomic" "time" @@ -515,10 +514,13 @@ func (bc *BlockChain) GetBodyRLP(hash common.Hash) rlp.RawValue { return body } -// HasBlock checks if a block is fully present in the database or not, caching -// it if present. -func (bc *BlockChain) HasBlock(hash common.Hash) bool { - return bc.GetBlockByHash(hash) != nil +// HasBlock checks if a block is fully present in the database or not. +func (bc *BlockChain) HasBlock(hash common.Hash, number uint64) bool { + if bc.blockCache.Contains(hash) { + return true + } + ok, _ := bc.chainDb.Has(blockBodyKey(hash, number)) + return ok } // HasBlockAndState checks if a block and associated state trie is fully present @@ -693,108 +695,73 @@ func SetReceiptsData(config *params.ChainConfig, block *types.Block, receipts ty // InsertReceiptChain attempts to complete an already existing header chain with // transaction and receipt data. -// XXX should this be moved to the test? func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { + bc.wg.Add(1) + defer bc.wg.Done() + // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(blockChain); i++ { if blockChain[i].NumberU64() != blockChain[i-1].NumberU64()+1 || blockChain[i].ParentHash() != blockChain[i-1].Hash() { - // Chain broke ancestry, log a messge (programming error) and skip insertion log.Error("Non contiguous receipt insert", "number", blockChain[i].Number(), "hash", blockChain[i].Hash(), "parent", blockChain[i].ParentHash(), "prevnumber", blockChain[i-1].Number(), "prevhash", blockChain[i-1].Hash()) - return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, blockChain[i-1].NumberU64(), blockChain[i-1].Hash().Bytes()[:4], i, blockChain[i].NumberU64(), blockChain[i].Hash().Bytes()[:4], blockChain[i].ParentHash().Bytes()[:4]) } } - // Pre-checks passed, start the block body and receipt imports - bc.wg.Add(1) - defer bc.wg.Done() - // Collect some import statistics to report on - stats := struct{ processed, ignored int32 }{} - start := time.Now() + var ( + stats = struct{ processed, ignored int32 }{} + start = time.Now() + bytes = 0 + batch = bc.chainDb.NewBatch() + ) + for i, block := range blockChain { + receipts := receiptChain[i] + // Short circuit insertion if shutting down or processing failed + if atomic.LoadInt32(&bc.procInterrupt) == 1 { + return 0, nil + } + // Short circuit if the owner header is unknown + if !bc.HasHeader(block.Hash(), block.NumberU64()) { + return i, fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4]) + } + // Skip if the entire data is already known + if bc.HasBlock(block.Hash(), block.NumberU64()) { + stats.ignored++ + continue + } + // Compute all the non-consensus fields of the receipts + SetReceiptsData(bc.config, block, receipts) + // Write all the data out into the database + if err := WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()); err != nil { + return i, fmt.Errorf("failed to write block body: %v", err) + } + if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { + return i, fmt.Errorf("failed to write block receipts: %v", err) + } + if err := WriteTxLookupEntries(batch, block); err != nil { + return i, fmt.Errorf("failed to write lookup metadata: %v", err) + } + stats.processed++ - // Create the block importing task queue and worker functions - tasks := make(chan int, len(blockChain)) - for i := 0; i < len(blockChain) && i < len(receiptChain); i++ { - tasks <- i - } - close(tasks) - - errs, failed := make([]error, len(tasks)), int32(0) - process := func(worker int) { - for index := range tasks { - block, receipts := blockChain[index], receiptChain[index] - - // Short circuit insertion if shutting down or processing failed - if atomic.LoadInt32(&bc.procInterrupt) == 1 { - return + if batch.ValueSize() >= ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + return 0, err } - if atomic.LoadInt32(&failed) > 0 { - return - } - // Short circuit if the owner header is unknown - if !bc.HasHeader(block.Hash()) { - errs[index] = fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4]) - atomic.AddInt32(&failed, 1) - return - } - // Skip if the entire data is already known - if bc.HasBlock(block.Hash()) { - atomic.AddInt32(&stats.ignored, 1) - continue - } - // Compute all the non-consensus fields of the receipts - SetReceiptsData(bc.config, block, receipts) - // Write all the data out into the database - if err := WriteBody(bc.chainDb, block.Hash(), block.NumberU64(), block.Body()); err != nil { - errs[index] = fmt.Errorf("failed to write block body: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write block body", "err", err) - return - } - if err := WriteBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { - errs[index] = fmt.Errorf("failed to write block receipts: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write block receipts", "err", err) - return - } - if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { - errs[index] = fmt.Errorf("failed to write lookup metadata: %v", err) - atomic.AddInt32(&failed, 1) - log.Crit("Failed to write lookup metadata", "err", err) - return - } - atomic.AddInt32(&stats.processed, 1) + bytes += batch.ValueSize() + batch = bc.chainDb.NewBatch() } } - // Start as many worker threads as goroutines allowed - pending := new(sync.WaitGroup) - for i := 0; i < runtime.GOMAXPROCS(0); i++ { - pending.Add(1) - go func(id int) { - defer pending.Done() - process(id) - }(i) - } - pending.Wait() - - // If anything failed, report - if failed > 0 { - for i, err := range errs { - if err != nil { - return i, err - } + if batch.ValueSize() > 0 { + bytes += batch.ValueSize() + if err := batch.Write(); err != nil { + return 0, err } } - if atomic.LoadInt32(&bc.procInterrupt) == 1 { - log.Debug("Premature abort during receipts processing") - return 0, nil - } + // Update the head fast sync block if better bc.mu.Lock() - - head := blockChain[len(errs)-1] + head := blockChain[len(blockChain)-1] if td := bc.GetTd(head.Hash(), head.NumberU64()); td != nil { // Rewind may have occurred, skip in that case if bc.GetTd(bc.currentFastBlock.Hash(), bc.currentFastBlock.NumberU64()).Cmp(td) < 0 { if err := WriteHeadFastBlockHash(bc.chainDb, head.Hash()); err != nil { @@ -805,16 +772,18 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } bc.mu.Unlock() - // Report some public statistics so the user has a clue what's going on - last := blockChain[len(blockChain)-1] - log.Info("Imported new block receipts", "count", stats.processed, "elapsed", common.PrettyDuration(time.Since(start)), - "number", last.Number(), "hash", last.Hash(), "ignored", stats.ignored) - + log.Info("Imported new block receipts", + "count", stats.processed, + "elapsed", common.PrettyDuration(time.Since(start)), + "bytes", bytes, + "number", head.Number(), + "hash", head.Hash(), + "ignored", stats.ignored) return 0, nil } // WriteBlock writes the block to the chain. -func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err error) { +func (bc *BlockChain) WriteBlockAndState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) { bc.wg.Add(1) defer bc.wg.Done() @@ -827,7 +796,7 @@ func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err er bc.mu.Lock() defer bc.mu.Unlock() - if bc.HasBlock(block.Hash()) { + if bc.HasBlock(block.Hash(), block.NumberU64()) { log.Trace("Block existed", "hash", block.Hash()) return } @@ -837,10 +806,18 @@ func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err er // Irrelevant of the canonical status, write the block itself to the database if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil { - log.Crit("Failed to write block total difficulty", "err", err) + return NonStatTy, err } - if err := WriteBlock(bc.chainDb, block); err != nil { - log.Crit("Failed to write block contents", "err", err) + // Write other block data using a batch. + batch := bc.chainDb.NewBatch() + if err := WriteBlock(batch, block); err != nil { + return NonStatTy, err + } + if _, err := state.CommitTo(batch, bc.config.IsEIP158(block.Number())); err != nil { + return NonStatTy, err + } + if err := WriteBlockReceipts(batch, block.Hash(), block.NumberU64(), receipts); err != nil { + return NonStatTy, err } // If the total difficulty is higher than our known, add it to the canonical chain @@ -853,20 +830,46 @@ func (bc *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err er return NonStatTy, err } } - bc.insert(block) // Insert the block as the new head of the chain + // Write the positional metadata for transaction and receipt lookups + if err := WriteTxLookupEntries(batch, block); err != nil { + return NonStatTy, err + } + // Write hash preimages + if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil { + return NonStatTy, err + } status = CanonStatTy } else { status = SideStatTy } + if err := batch.Write(); err != nil { + return NonStatTy, err + } + // Set new head. + if status == CanonStatTy { + bc.insert(block) + } bc.futureBlocks.Remove(block.Hash()) - - return + return status, nil } -// InsertChain will attempt to insert the given chain in to the canonical chain or, otherwise, create a fork. If an error is returned -// it will return the index number of the failing block as well an error describing what went wrong (for possible errors see core/errors.go). +// InsertChain attempts to insert the given batch of blocks in to the canonical +// chain or, otherwise, create a fork. If an error is returned it will return +// the index number of the failing block as well an error describing what went +// wrong. +// +// After insertion is done, all accumulated events will be fired. func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { + n, events, logs, err := bc.insertChain(chain) + bc.PostChainEvents(events, logs) + return n, err +} + +// insertChain will execute the actual chain insertion and event aggregation. The +// only reason this method exists as a separate one is to make locking cleaner +// with deferred statements. +func (bc *BlockChain) insertChain(chain types.Blocks) (int, []interface{}, []*types.Log, error) { // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(chain); i++ { if chain[i].NumberU64() != chain[i-1].NumberU64()+1 || chain[i].ParentHash() != chain[i-1].Hash() { @@ -874,7 +877,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { log.Error("Non contiguous block insert", "number", chain[i].Number(), "hash", chain[i].Hash(), "parent", chain[i].ParentHash(), "prevnumber", chain[i-1].Number(), "prevhash", chain[i-1].Hash()) - return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(), + return 0, nil, nil, fmt.Errorf("non contiguous insert: item %d is #%d [%x…], item %d is #%d [%x…] (parent [%x…])", i-1, chain[i-1].NumberU64(), chain[i-1].Hash().Bytes()[:4], i, chain[i].NumberU64(), chain[i].Hash().Bytes()[:4], chain[i].ParentHash().Bytes()[:4]) } } @@ -891,6 +894,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { var ( stats = insertStats{startTime: mclock.Now()} events = make([]interface{}, 0, len(chain)) + lastCanon *types.Block coalescedLogs []*types.Log ) // Start the parallel header verifier @@ -914,7 +918,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // If the header is a banned one, straight out abort if BadHashes[block.Hash()] { bc.reportBlock(block, nil, ErrBlacklistedHash) - return i, ErrBlacklistedHash + return i, events, coalescedLogs, ErrBlacklistedHash } // Wait for the block's verification to complete bstart := time.Now() @@ -935,7 +939,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // if given. max := big.NewInt(time.Now().Unix() + maxTimeFutureBlocks) if block.Time().Cmp(max) > 0 { - return i, fmt.Errorf("future block: %v > %v", block.Time(), max) + return i, events, coalescedLogs, fmt.Errorf("future block: %v > %v", block.Time(), max) } bc.futureBlocks.Add(block.Hash(), block) stats.queued++ @@ -949,7 +953,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { } bc.reportBlock(block, nil, err) - return i, err + return i, events, coalescedLogs, err } // Create a new statedb using the parent block and report an // error if it fails. @@ -961,60 +965,35 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { } state, err := state.New(parent.Root(), bc.stateCache) if err != nil { - return i, err + return i, events, coalescedLogs, err } // Process block using the parent state as reference point. receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig) if err != nil { bc.reportBlock(block, receipts, err) - return i, err + return i, events, coalescedLogs, err } // Validate the state using the default validator err = bc.Validator().ValidateState(block, parent, state, receipts, usedGas) if err != nil { bc.reportBlock(block, receipts, err) - return i, err + return i, events, coalescedLogs, err } - // Write state changes to database - if _, err = state.CommitTo(bc.chainDb, bc.config.IsEIP158(block.Number())); err != nil { - return i, err - } - - // coalesce logs for later processing - coalescedLogs = append(coalescedLogs, logs...) - - if err = WriteBlockReceipts(bc.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { - return i, err - } - - // write the block to the chain and get the status - status, err := bc.WriteBlock(block) + // Write the block to the chain and get the status. + status, err := bc.WriteBlockAndState(block, receipts, state) if err != nil { - return i, err + return i, events, coalescedLogs, err } - switch status { case CanonStatTy: log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), "uncles", len(block.Uncles()), "txs", len(block.Transactions()), "gas", block.GasUsed(), "elapsed", common.PrettyDuration(time.Since(bstart))) + coalescedLogs = append(coalescedLogs, logs...) blockInsertTimer.UpdateSince(bstart) events = append(events, ChainEvent{block, block.Hash(), logs}) - // We need some control over the mining operation. Acquiring locks and waiting - // for the miner to create new block takes too long and in most cases isn't - // even necessary. - if bc.LastBlockHash() == block.Hash() { - events = append(events, ChainHeadEvent{block}) - } + lastCanon = block - // Write the positional metadata for transaction and receipt lookups - if err := WriteTxLookupEntries(bc.chainDb, block); err != nil { - return i, err - } - // Write hash preimages - if err := WritePreimages(bc.chainDb, block.NumberU64(), state.Preimages()); err != nil { - return i, err - } case SideStatTy: log.Debug("Inserted forked block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(bstart)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles())) @@ -1026,9 +1005,11 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { stats.usedGas += usedGas.Uint64() stats.report(chain, i) } - go bc.PostChainEvents(events, coalescedLogs) - - return 0, nil + // Append a single chain head event if we've progressed the chain + if lastCanon != nil && bc.LastBlockHash() == lastCanon.Hash() { + events = append(events, ChainHeadEvent{lastCanon}) + } + return 0, events, coalescedLogs, nil } // insertStats tracks and reports on block insertion. @@ -1357,8 +1338,8 @@ func (bc *BlockChain) GetHeaderByHash(hash common.Hash) *types.Header { // HasHeader checks if a block header is present in the database or not, caching // it if present. -func (bc *BlockChain) HasHeader(hash common.Hash) bool { - return bc.hc.HasHeader(hash) +func (bc *BlockChain) HasHeader(hash common.Hash, number uint64) bool { + return bc.hc.HasHeader(hash, number) } // GetBlockHashesFromHash retrieves a number of block hashes starting at a given diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 470974a0a7..cb1df8d4b0 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -928,14 +928,14 @@ func TestReorgSideEvent(t *testing.T) { replacementBlocks, _ := GenerateChain(gspec.Config, genesis, db, 4, func(i int, gen *BlockGen) { tx, err := types.SignTx(types.NewContractCreation(gen.TxNonce(addr1), new(big.Int), big.NewInt(1000000), new(big.Int), nil), signer, key1) if i == 2 { - gen.OffsetTime(-1) + gen.OffsetTime(-9) } if err != nil { t.Fatalf("failed to create tx: %v", err) } gen.AddTx(tx) }) - chainSideCh := make(chan ChainSideEvent) + chainSideCh := make(chan ChainSideEvent, 64) blockchain.SubscribeChainSideEvent(chainSideCh) if _, err := blockchain.InsertChain(replacementBlocks); err != nil { t.Fatalf("failed to insert chain: %v", err) diff --git a/core/dao.go b/core/dao.go deleted file mode 100644 index ff42a0e9d3..0000000000 --- a/core/dao.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "bytes" - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/params" -) - -// ValidateDAOHeaderExtraData validates the extra-data field of a block header to -// ensure it conforms to DAO hard-fork rules. -// -// DAO hard-fork extension to the header validity: -// a) if the node is no-fork, do not accept blocks in the [fork, fork+10) range -// with the fork specific extra-data set -// b) if the node is pro-fork, require blocks in the specific range to have the -// unique extra-data set. -func ValidateDAOHeaderExtraData(config *params.ChainConfig, header *types.Header) error { - // Short circuit validation if the node doesn't care about the DAO fork - if config.DAOForkBlock == nil { - return nil - } - // Make sure the block is within the fork's modified extra-data range - limit := new(big.Int).Add(config.DAOForkBlock, params.DAOForkExtraRange) - if header.Number.Cmp(config.DAOForkBlock) < 0 || header.Number.Cmp(limit) >= 0 { - return nil - } - // Depending whether we support or oppose the fork, validate the extra-data contents - if config.DAOForkSupport { - if !bytes.Equal(header.Extra, params.DAOForkBlockExtra) { - return fmt.Errorf("DAO pro-fork bad block extra-data: 0x%x", header.Extra) - } - } else { - if bytes.Equal(header.Extra, params.DAOForkBlockExtra) { - return fmt.Errorf("DAO no-fork bad block extra-data: 0x%x", header.Extra) - } - } - // All ok, header has the same extra-data we expect - return nil -} - -// ApplyDAOHardFork modifies the state database according to the DAO hard-fork -// rules, transferring all balances of a set of DAO accounts to a single refund -// contract. -func ApplyDAOHardFork(statedb *state.StateDB) { - // Retrieve the contract to refund balances into - if !statedb.Exist(params.DAORefundContract) { - statedb.CreateAccount(params.DAORefundContract) - } - - // Move every DAO account and extra-balance account funds into the refund contract - for _, addr := range params.DAODrainList() { - statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr)) - statedb.SetBalance(addr, new(big.Int)) - } -} diff --git a/core/dao_test.go b/core/dao_test.go index d6e11d78ae..b9898ff7c8 100644 --- a/core/dao_test.go +++ b/core/dao_test.go @@ -40,14 +40,22 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create the concurrent, conflicting two nodes proDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(proDb) - proConf := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: true} - proBc, _ := NewBlockChain(proDb, proConf, ethash.NewFaker(), vm.Config{}) + + proConf := *params.TestChainConfig + proConf.DAOForkBlock = forkBlock + proConf.DAOForkSupport = true + + proBc, _ := NewBlockChain(proDb, &proConf, ethash.NewFaker(), vm.Config{}) defer proBc.Stop() conDb, _ := ethdb.NewMemDatabase() gspec.MustCommit(conDb) - conConf := ¶ms.ChainConfig{HomesteadBlock: big.NewInt(0), DAOForkBlock: forkBlock, DAOForkSupport: false} - conBc, _ := NewBlockChain(conDb, conConf, ethash.NewFaker(), vm.Config{}) + + conConf := *params.TestChainConfig + conConf.DAOForkBlock = forkBlock + conConf.DAOForkSupport = false + + conBc, _ := NewBlockChain(conDb, &conConf, ethash.NewFaker(), vm.Config{}) defer conBc.Stop() if _, err := proBc.InsertChain(prefix); err != nil { @@ -61,7 +69,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Create a pro-fork block, and try to feed into the no-fork chain db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, conConf, ethash.NewFaker(), vm.Config{}) + bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -71,19 +79,19 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err == nil { t.Fatalf("contra-fork chain accepted pro-fork block: %v", blocks[0]) } // Create a proper no-fork block for the contra-forker - blocks, _ = GenerateChain(conConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err != nil { t.Fatalf("contra-fork chain didn't accepted no-fork block: %v", err) } // Create a no-fork block, and try to feed into the pro-fork chain db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, proConf, ethash.NewFaker(), vm.Config{}) + bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -93,12 +101,12 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err == nil { t.Fatalf("pro-fork chain accepted contra-fork block: %v", blocks[0]) } // Create a proper pro-fork block for the pro-forker - blocks, _ = GenerateChain(proConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err != nil { t.Fatalf("pro-fork chain didn't accepted pro-fork block: %v", err) } @@ -106,7 +114,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { // Verify that contra-forkers accept pro-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ := NewBlockChain(db, conConf, ethash.NewFaker(), vm.Config{}) + bc, _ := NewBlockChain(db, &conConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks := conBc.GetBlocksFromHash(conBc.CurrentBlock().Hash(), int(conBc.CurrentBlock().NumberU64())) @@ -116,14 +124,14 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import contra-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := conBc.InsertChain(blocks); err != nil { t.Fatalf("contra-fork chain didn't accept pro-fork block post-fork: %v", err) } // Verify that pro-forkers accept contra-fork extra-datas after forking finishes db, _ = ethdb.NewMemDatabase() gspec.MustCommit(db) - bc, _ = NewBlockChain(db, proConf, ethash.NewFaker(), vm.Config{}) + bc, _ = NewBlockChain(db, &proConf, ethash.NewFaker(), vm.Config{}) defer bc.Stop() blocks = proBc.GetBlocksFromHash(proBc.CurrentBlock().Hash(), int(proBc.CurrentBlock().NumberU64())) @@ -133,7 +141,7 @@ func TestDAOForkRangeExtradata(t *testing.T) { if _, err := bc.InsertChain(blocks); err != nil { t.Fatalf("failed to import pro-fork chain for expansion: %v", err) } - blocks, _ = GenerateChain(conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) + blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), db, 1, func(i int, gen *BlockGen) {}) if _, err := proBc.InsertChain(blocks); err != nil { t.Fatalf("pro-fork chain didn't accept contra-fork block post-fork: %v", err) } diff --git a/core/database_util.go b/core/database_util.go index 989071104e..1730a048e7 100644 --- a/core/database_util.go +++ b/core/database_util.go @@ -38,11 +38,6 @@ type DatabaseReader interface { Get(key []byte) (value []byte, err error) } -// DatabaseWriter wraps the Put method of a backing data store. -type DatabaseWriter interface { - Put(key, value []byte) error -} - // DatabaseDeleter wraps the Delete method of a backing data store. type DatabaseDeleter interface { Delete(key []byte) error @@ -154,7 +149,7 @@ func GetHeadFastBlockHash(db DatabaseReader) common.Hash { // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil // if the header's not found. func GetHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + data, _ := db.Get(headerKey(hash, number)) return data } @@ -175,10 +170,18 @@ func GetHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Header // GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func GetBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + data, _ := db.Get(blockBodyKey(hash, number)) return data } +func headerKey(hash common.Hash, number uint64) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +func blockBodyKey(hash common.Hash, number uint64) []byte { + return append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + // GetBody retrieves the block body (transactons, uncles) corresponding to the // hash, nil if none found. func GetBody(db DatabaseReader, hash common.Hash, number uint64) *types.Body { @@ -340,7 +343,7 @@ func GetBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) } // WriteCanonicalHash stores the canonical hash for the given block number. -func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) error { +func WriteCanonicalHash(db ethdb.Putter, hash common.Hash, number uint64) error { key := append(append(headerPrefix, encodeBlockNumber(number)...), numSuffix...) if err := db.Put(key, hash.Bytes()); err != nil { log.Crit("Failed to store number to hash mapping", "err", err) @@ -349,7 +352,7 @@ func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) erro } // WriteHeadHeaderHash stores the head header's hash. -func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) error { +func WriteHeadHeaderHash(db ethdb.Putter, hash common.Hash) error { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { log.Crit("Failed to store last header's hash", "err", err) } @@ -357,7 +360,7 @@ func WriteHeadHeaderHash(db DatabaseWriter, hash common.Hash) error { } // WriteHeadBlockHash stores the head block's hash. -func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) error { +func WriteHeadBlockHash(db ethdb.Putter, hash common.Hash) error { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { log.Crit("Failed to store last block's hash", "err", err) } @@ -365,7 +368,7 @@ func WriteHeadBlockHash(db DatabaseWriter, hash common.Hash) error { } // WriteHeadFastBlockHash stores the fast head block's hash. -func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) error { +func WriteHeadFastBlockHash(db ethdb.Putter, hash common.Hash) error { if err := db.Put(headFastKey, hash.Bytes()); err != nil { log.Crit("Failed to store last fast block's hash", "err", err) } @@ -373,7 +376,7 @@ func WriteHeadFastBlockHash(db DatabaseWriter, hash common.Hash) error { } // WriteHeader serializes a block header into the database. -func WriteHeader(db DatabaseWriter, header *types.Header) error { +func WriteHeader(db ethdb.Putter, header *types.Header) error { data, err := rlp.EncodeToBytes(header) if err != nil { return err @@ -393,7 +396,7 @@ func WriteHeader(db DatabaseWriter, header *types.Header) error { } // WriteBody serializes the body of a block into the database. -func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.Body) error { +func WriteBody(db ethdb.Putter, hash common.Hash, number uint64, body *types.Body) error { data, err := rlp.EncodeToBytes(body) if err != nil { return err @@ -402,7 +405,7 @@ func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.B } // WriteBodyRLP writes a serialized body of a block into the database. -func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.RawValue) error { +func WriteBodyRLP(db ethdb.Putter, hash common.Hash, number uint64, rlp rlp.RawValue) error { key := append(append(bodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) if err := db.Put(key, rlp); err != nil { log.Crit("Failed to store block body", "err", err) @@ -411,7 +414,7 @@ func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.Ra } // WriteTd serializes the total difficulty of a block into the database. -func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) error { +func WriteTd(db ethdb.Putter, hash common.Hash, number uint64, td *big.Int) error { data, err := rlp.EncodeToBytes(td) if err != nil { return err @@ -424,7 +427,7 @@ func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) er } // WriteBlock serializes a block into the database, header and body separately. -func WriteBlock(db DatabaseWriter, block *types.Block) error { +func WriteBlock(db ethdb.Putter, block *types.Block) error { // Store the body first to retain database consistency if err := WriteBody(db, block.Hash(), block.NumberU64(), block.Body()); err != nil { return err @@ -439,7 +442,7 @@ func WriteBlock(db DatabaseWriter, block *types.Block) error { // WriteBlockReceipts stores all the transaction receipts belonging to a block // as a single receipt slice. This is used during chain reorganisations for // rescheduling dropped transactions. -func WriteBlockReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts types.Receipts) error { +func WriteBlockReceipts(db ethdb.Putter, hash common.Hash, number uint64, receipts types.Receipts) error { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { @@ -459,9 +462,7 @@ func WriteBlockReceipts(db DatabaseWriter, hash common.Hash, number uint64, rece // WriteTxLookupEntries stores a positional metadata for every transaction from // a block, enabling hash based transaction and receipt lookups. -func WriteTxLookupEntries(db ethdb.Database, block *types.Block) error { - batch := db.NewBatch() - +func WriteTxLookupEntries(db ethdb.Putter, block *types.Block) error { // Iterate over each transaction and encode its metadata for i, tx := range block.Transactions() { entry := txLookupEntry{ @@ -473,20 +474,16 @@ func WriteTxLookupEntries(db ethdb.Database, block *types.Block) error { if err != nil { return err } - if err := batch.Put(append(lookupPrefix, tx.Hash().Bytes()...), data); err != nil { + if err := db.Put(append(lookupPrefix, tx.Hash().Bytes()...), data); err != nil { return err } } - // Write the scheduled data into the database - if err := batch.Write(); err != nil { - log.Crit("Failed to store lookup entries", "err", err) - } return nil } // WriteBloomBits writes the compressed bloom bits vector belonging to the given // section and bit index. -func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) { +func WriteBloomBits(db ethdb.Putter, bit uint, section uint64, head common.Hash, bits []byte) { key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) binary.BigEndian.PutUint16(key[1:], uint16(bit)) @@ -572,13 +569,13 @@ func GetBlockChainVersion(db DatabaseReader) int { } // WriteBlockChainVersion writes vsn as the version number to db. -func WriteBlockChainVersion(db DatabaseWriter, vsn int) { +func WriteBlockChainVersion(db ethdb.Putter, vsn int) { enc, _ := rlp.EncodeToBytes(uint(vsn)) db.Put([]byte("BlockchainVersion"), enc) } // WriteChainConfig writes the chain config settings to the database. -func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConfig) error { +func WriteChainConfig(db ethdb.Putter, hash common.Hash, cfg *params.ChainConfig) error { // short circuit and ignore if nil config. GetChainConfig // will return a default. if cfg == nil { diff --git a/core/headerchain.go b/core/headerchain.go index 6ec44b61d5..0e5215293d 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -267,7 +267,7 @@ func (hc *HeaderChain) InsertHeaderChain(chain []*types.Header, writeHeader WhCa return i, errors.New("aborted") } // If the header's already known, skip it, otherwise store - if hc.GetHeader(header.Hash(), header.Number.Uint64()) != nil { + if hc.HasHeader(header.Hash(), header.Number.Uint64()) { stats.ignored++ continue } @@ -361,10 +361,13 @@ func (hc *HeaderChain) GetHeaderByHash(hash common.Hash) *types.Header { return hc.GetHeader(hash, hc.GetBlockNumber(hash)) } -// HasHeader checks if a block header is present in the database or not, caching -// it if present. -func (hc *HeaderChain) HasHeader(hash common.Hash) bool { - return hc.GetHeaderByHash(hash) != nil +// HasHeader checks if a block header is present in the database or not. +func (hc *HeaderChain) HasHeader(hash common.Hash, number uint64) bool { + if hc.numberCache.Contains(hash) || hc.headerCache.Contains(hash) { + return true + } + ok, _ := hc.chainDb.Has(headerKey(hash, number)) + return ok } // GetHeaderByNumber retrieves a block header from the database by number, diff --git a/core/state/sync.go b/core/state/sync.go index 2c29d706aa..28fcf6ae05 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -18,60 +18,24 @@ package state import ( "bytes" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) -// StateSync is the main state synchronisation scheduler, which provides yet the -// unknown state hashes to retrieve, accepts node data associated with said hashes -// and reconstructs the state database step by step until all is done. -type StateSync trie.TrieSync - // NewStateSync create a new state trie download scheduler. -func NewStateSync(root common.Hash, database trie.DatabaseReader) *StateSync { +func NewStateSync(root common.Hash, database trie.DatabaseReader) *trie.TrieSync { var syncer *trie.TrieSync - callback := func(leaf []byte, parent common.Hash) error { - var obj struct { - Nonce uint64 - Balance *big.Int - Root common.Hash - CodeHash []byte - } + var obj Account if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { return err } syncer.AddSubTrie(obj.Root, 64, parent, nil) syncer.AddRawEntry(common.BytesToHash(obj.CodeHash), 64, parent) - return nil } syncer = trie.NewTrieSync(root, database, callback) - return (*StateSync)(syncer) -} - -// Missing retrieves the known missing nodes from the state trie for retrieval. -func (s *StateSync) Missing(max int) []common.Hash { - return (*trie.TrieSync)(s).Missing(max) -} - -// Process injects a batch of retrieved trie nodes data, returning if something -// was committed to the memcache and also the index of an entry if processing of -// it failed. -func (s *StateSync) Process(list []trie.SyncResult) (bool, int, error) { - return (*trie.TrieSync)(s).Process(list) -} - -// Commit flushes the data stored in the internal memcache out to persistent -// storage, returning th enumber of items written and any occurred error. -func (s *StateSync) Commit(dbw trie.DatabaseWriter) (int, error) { - return (*trie.TrieSync)(s).Commit(dbw) -} - -// Pending returns the number of state entries currently pending for download. -func (s *StateSync) Pending() int { - return (*trie.TrieSync)(s).Pending() + return syncer } diff --git a/core/state_processor.go b/core/state_processor.go index 4115eab8cb..689c83785f 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -105,7 +105,7 @@ func ApplyTransaction(config *params.ChainConfig, bc *BlockChain, author *common // Update the state with pending changes var root []byte - if config.IsMetropolis(header.Number) { + if config.IsByzantium(header.Number) { statedb.Finalise(true) } else { root = statedb.IntermediateRoot(config.IsEIP158(header.Number)).Bytes() diff --git a/core/types/receipt.go b/core/types/receipt.go index 4f3b7357ed..e179fe0cfd 100644 --- a/core/types/receipt.go +++ b/core/types/receipt.go @@ -79,7 +79,7 @@ func NewReceipt(root []byte, failed bool, cumulativeGasUsed *big.Int) *Receipt { } // EncodeRLP implements rlp.Encoder, and flattens the consensus fields of a receipt -// into an RLP stream. If no post state is present, metropolis fork is assumed. +// into an RLP stream. If no post state is present, byzantium fork is assumed. func (r *Receipt) EncodeRLP(w io.Writer) error { return rlp.Encode(w, &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs}) } diff --git a/core/types/transaction.go b/core/types/transaction.go index 947fc85d60..7f54860fc0 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -381,28 +381,32 @@ func (s *TxByPrice) Pop() interface{} { // transactions in a profit-maximising sorted order, while supporting removing // entire batches of transactions for non-executable accounts. type TransactionsByPriceAndNonce struct { - txs map[common.Address]Transactions // Per account nonce-sorted list of transactions - heads TxByPrice // Next transaction for each unique account (price heap) + txs map[common.Address]Transactions // Per account nonce-sorted list of transactions + heads TxByPrice // Next transaction for each unique account (price heap) + signer Signer // Signer for the set of transactions } // NewTransactionsByPriceAndNonce creates a transaction set that can retrieve // price sorted transactions in a nonce-honouring way. // // Note, the input map is reowned so the caller should not interact any more with -// if after providng it to the constructor. -func NewTransactionsByPriceAndNonce(txs map[common.Address]Transactions) *TransactionsByPriceAndNonce { +// if after providing it to the constructor. +func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions) *TransactionsByPriceAndNonce { // Initialize a price based heap with the head transactions heads := make(TxByPrice, 0, len(txs)) - for acc, accTxs := range txs { + for _, accTxs := range txs { heads = append(heads, accTxs[0]) + // Ensure the sender address is from the signer + acc, _ := Sender(signer, accTxs[0]) txs[acc] = accTxs[1:] } heap.Init(&heads) // Assemble and return the transaction set return &TransactionsByPriceAndNonce{ - txs: txs, - heads: heads, + txs: txs, + heads: heads, + signer: signer, } } @@ -416,9 +420,7 @@ func (t *TransactionsByPriceAndNonce) Peek() *Transaction { // Shift replaces the current best head with the next one from the same account. func (t *TransactionsByPriceAndNonce) Shift() { - signer := deriveSigner(t.heads[0].data.V) - // derive signer but don't cache. - acc, _ := Sender(signer, t.heads[0]) // we only sort valid txs so this cannot fail + acc, _ := Sender(t.signer, t.heads[0]) if txs, ok := t.txs[acc]; ok && len(txs) > 0 { t.heads[0], t.txs[acc] = txs[0], txs[1:] heap.Fix(&t.heads, 0) diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index edbe171e6f..df9d7ffd18 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -143,7 +143,7 @@ func TestTransactionPriceNonceSort(t *testing.T) { } } // Sort the transactions and cross check the nonce ordering - txset := NewTransactionsByPriceAndNonce(groups) + txset := NewTransactionsByPriceAndNonce(signer, groups) txs := Transactions{} for { diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 790d42bbe6..7344b6043c 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -46,9 +46,9 @@ var PrecompiledContractsHomestead = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{4}): &dataCopy{}, } -// PrecompiledContractsMetropolis contains the default set of pre-compiled Ethereum -// contracts used in the Metropolis release. -var PrecompiledContractsMetropolis = map[common.Address]PrecompiledContract{ +// PrecompiledContractsByzantium contains the default set of pre-compiled Ethereum +// contracts used in the Byzantium release. +var PrecompiledContractsByzantium = map[common.Address]PrecompiledContract{ common.BytesToAddress([]byte{1}): &ecrecover{}, common.BytesToAddress([]byte{2}): &sha256hash{}, common.BytesToAddress([]byte{3}): &ripemd160hash{}, diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 880d7324ae..513651835a 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -321,7 +321,7 @@ var bn256PairingTests = []precompiledTest{ } func testPrecompiled(addr string, test precompiledTest, t *testing.T) { - p := PrecompiledContractsMetropolis[common.HexToAddress(addr)] + p := PrecompiledContractsByzantium[common.HexToAddress(addr)] in := common.Hex2Bytes(test.input) contract := NewContract(AccountRef(common.HexToAddress("1337")), nil, new(big.Int), p.RequiredGas(in)) @@ -338,7 +338,7 @@ func benchmarkPrecompiled(addr string, test precompiledTest, bench *testing.B) { if test.noBenchmark { return } - p := PrecompiledContractsMetropolis[common.HexToAddress(addr)] + p := PrecompiledContractsByzantium[common.HexToAddress(addr)] in := common.Hex2Bytes(test.input) reqGas := p.RequiredGas(in) contract := NewContract(AccountRef(common.HexToAddress("1337")), diff --git a/core/vm/evm.go b/core/vm/evm.go index 495d9beea1..093c7d4c14 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -41,8 +41,8 @@ type ( func run(evm *EVM, snapshot int, contract *Contract, input []byte) ([]byte, error) { if contract.CodeAddr != nil { precompiles := PrecompiledContractsHomestead - if evm.ChainConfig().IsMetropolis(evm.BlockNumber) { - precompiles = PrecompiledContractsMetropolis + if evm.ChainConfig().IsByzantium(evm.BlockNumber) { + precompiles = PrecompiledContractsByzantium } if p := precompiles[*contract.CodeAddr]; p != nil { return RunPrecompiledContract(p, input, contract) @@ -151,8 +151,8 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas ) if !evm.StateDB.Exist(addr) { precompiles := PrecompiledContractsHomestead - if evm.ChainConfig().IsMetropolis(evm.BlockNumber) { - precompiles = PrecompiledContractsMetropolis + if evm.ChainConfig().IsByzantium(evm.BlockNumber) { + precompiles = PrecompiledContractsByzantium } if precompiles[addr] == nil && evm.ChainConfig().IsEIP158(evm.BlockNumber) && value.Sign() == 0 { return nil, gas, nil @@ -299,9 +299,6 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // Create creates a new contract using code as deployment code. func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { - if evm.vmConfig.NoRecursion && evm.depth > 0 { - return nil, common.Address{}, gas, nil - } // Depth check execution. Fail if we're trying to execute above the // limit. @@ -334,6 +331,9 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I contract := NewContract(caller, AccountRef(contractAddr), value, gas) contract.SetCallCode(&contractAddr, crypto.Keccak256Hash(code), code) + if evm.vmConfig.NoRecursion && evm.depth > 0 { + return nil, contractAddr, gas, nil + } ret, err = run(evm, snapshot, contract, nil) // check whether the max code size has been exceeded maxCodeSizeExceeded := evm.ChainConfig().IsEIP158(evm.BlockNumber) && len(ret) > params.MaxCodeSize diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 13712295c9..0d8e295a52 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -324,7 +324,7 @@ func gasCall(gt params.GasTable, evm *EVM, contract *Contract, stack *Stack, mem eip158 = evm.ChainConfig().IsEIP158(evm.BlockNumber) ) if eip158 { - if transfersValue && evm.StateDB.Empty(address) { + if transfersValue && evm.StateDB.Empty(address) { gas += params.CallNewAccountGas } } else if !evm.StateDB.Exist(address) { diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index d3e24a7a45..b0d796a44b 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -70,8 +70,8 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter { // we'll set the default jump table. if !cfg.JumpTable[STOP].valid { switch { - case evm.ChainConfig().IsMetropolis(evm.BlockNumber): - cfg.JumpTable = metropolisInstructionSet + case evm.ChainConfig().IsByzantium(evm.BlockNumber): + cfg.JumpTable = byzantiumInstructionSet case evm.ChainConfig().IsHomestead(evm.BlockNumber): cfg.JumpTable = homesteadInstructionSet default: @@ -88,7 +88,7 @@ func NewInterpreter(evm *EVM, cfg Config) *Interpreter { } func (in *Interpreter) enforceRestrictions(op OpCode, operation operation, stack *Stack) error { - if in.evm.chainRules.IsMetropolis { + if in.evm.chainRules.IsByzantium { if in.readOnly { // If the interpreter is operating in readonly mode, make sure no // state-modifying operation is performed. The 3rd stack item diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index f0a922912f..9ef192fdf9 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -51,14 +51,14 @@ type operation struct { } var ( - frontierInstructionSet = NewFrontierInstructionSet() - homesteadInstructionSet = NewHomesteadInstructionSet() - metropolisInstructionSet = NewMetropolisInstructionSet() + frontierInstructionSet = NewFrontierInstructionSet() + homesteadInstructionSet = NewHomesteadInstructionSet() + byzantiumInstructionSet = NewByzantiumInstructionSet() ) -// NewMetropolisInstructionSet returns the frontier, homestead and -// metropolis instructions. -func NewMetropolisInstructionSet() [256]operation { +// NewByzantiumInstructionSet returns the frontier, homestead and +// byzantium instructions. +func NewByzantiumInstructionSet() [256]operation { // instructions that can be executed during the homestead phase. instructionSet := NewHomesteadInstructionSet() instructionSet[STATICCALL] = operation{ diff --git a/eth/api.go b/eth/api.go index 9904c6f536..a5b6e7076c 100644 --- a/eth/api.go +++ b/eth/api.go @@ -241,7 +241,7 @@ func (api *PrivateAdminAPI) ExportChain(file string) (bool, error) { func hasAllBlocks(chain *core.BlockChain, bs []*types.Block) bool { for _, b := range bs { - if !chain.HasBlock(b.Hash()) { + if !chain.HasBlock(b.Hash(), b.NumberU64()) { return false } } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 6ac58140a3..5782d4cf53 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -156,7 +156,7 @@ type Downloader struct { // LightChain encapsulates functions required to synchronise a light chain. type LightChain interface { // HasHeader verifies a header's presence in the local chain. - HasHeader(common.Hash) bool + HasHeader(h common.Hash, number uint64) bool // GetHeaderByHash retrieves a header from the local chain. GetHeaderByHash(common.Hash) *types.Header @@ -666,7 +666,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err continue } // Otherwise check if we already know the header or not - if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash())) { + if (d.mode == FullSync && d.blockchain.HasBlockAndState(headers[i].Hash())) || (d.mode != FullSync && d.lightchain.HasHeader(headers[i].Hash(), headers[i].Number.Uint64())) { number, hash = headers[i].Number.Uint64(), headers[i].Hash() // If every header is known, even future ones, the peer straight out lied about its head @@ -731,7 +731,7 @@ func (d *Downloader) findAncestor(p *peerConnection, height uint64) (uint64, err arrived = true // Modify the search interval based on the response - if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash())) { + if (d.mode == FullSync && !d.blockchain.HasBlockAndState(headers[0].Hash())) || (d.mode != FullSync && !d.lightchain.HasHeader(headers[0].Hash(), headers[0].Number.Uint64())) { end = check break } @@ -1093,6 +1093,10 @@ func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliv throttled = true break } + // Short circuit if there is no more available task. + if pending() == 0 { + break + } // Reserve a chunk of fetches for a peer. A nil can mean either that // no more headers are available, or that the peer is known not to // have them. @@ -1170,7 +1174,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { // If we're already past the pivot point, this could be an attack, thread carefully if rollback[len(rollback)-1].Number.Uint64() > pivot { - // If we didn't ever fail, lock in te pivot header (must! not! change!) + // If we didn't ever fail, lock in the pivot header (must! not! change!) if atomic.LoadUint32(&d.fsPivotFails) == 0 { for _, header := range rollback { if header.Number.Uint64() == pivot { @@ -1256,7 +1260,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { // Collect the yet unknown headers to mark them as uncertain unknown := make([]*types.Header, 0, len(headers)) for _, header := range chunk { - if !d.lightchain.HasHeader(header.Hash()) { + if !d.lightchain.HasHeader(header.Hash(), header.Number.Uint64()) { unknown = append(unknown, header) } } @@ -1392,7 +1396,6 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error { stateSync.Cancel() if err := d.commitPivotBlock(P); err != nil { return err - } } if err := d.importBlockResults(afterP); err != nil { diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index d66aafe94e..58f6e9a624 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -217,7 +217,7 @@ func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { } // HasHeader checks if a header is present in the testers canonical chain. -func (dl *downloadTester) HasHeader(hash common.Hash) bool { +func (dl *downloadTester) HasHeader(hash common.Hash, number uint64) bool { return dl.GetHeaderByHash(hash) != nil } diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index d0dc9a8aa6..e638744ea5 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -349,9 +349,10 @@ func (p *peerConnection) Lacks(hash common.Hash) bool { // peerSet represents the collection of active peer participating in the chain // download procedure. type peerSet struct { - peers map[string]*peerConnection - newPeerFeed event.Feed - lock sync.RWMutex + peers map[string]*peerConnection + newPeerFeed event.Feed + peerDropFeed event.Feed + lock sync.RWMutex } // newPeerSet creates a new peer set top track the active download sources. @@ -361,10 +362,16 @@ func newPeerSet() *peerSet { } } +// SubscribeNewPeers subscribes to peer arrival events. func (ps *peerSet) SubscribeNewPeers(ch chan<- *peerConnection) event.Subscription { return ps.newPeerFeed.Subscribe(ch) } +// SubscribePeerDrops subscribes to peer departure events. +func (ps *peerSet) SubscribePeerDrops(ch chan<- *peerConnection) event.Subscription { + return ps.peerDropFeed.Subscribe(ch) +} + // Reset iterates over the current peer set, and resets each of the known peers // to prepare for a next batch of block retrieval. func (ps *peerSet) Reset() { @@ -419,12 +426,15 @@ func (ps *peerSet) Register(p *peerConnection) error { // actions to/from that particular entity. func (ps *peerSet) Unregister(id string) error { ps.lock.Lock() - defer ps.lock.Unlock() - - if _, ok := ps.peers[id]; !ok { + p, ok := ps.peers[id] + if !ok { + defer ps.lock.Unlock() return errNotRegistered } delete(ps.peers, id) + ps.lock.Unlock() + + ps.peerDropFeed.Send(p) return nil } diff --git a/eth/downloader/statesync.go b/eth/downloader/statesync.go index a5ce8c42df..a0b05c9be6 100644 --- a/eth/downloader/statesync.go +++ b/eth/downloader/statesync.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" ) @@ -39,6 +40,7 @@ type stateReq struct { timer *time.Timer // Timer to fire when the RTT timeout expires peer *peerConnection // Peer that we're requesting from response [][]byte // Response data of the peer (nil for timeouts) + dropped bool // Flag whether the peer dropped off early } // timedOut returns if this request timed out. @@ -104,6 +106,11 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { go s.run() defer s.Cancel() + // Listen for peer departure events to cancel assigned tasks + peerDrop := make(chan *peerConnection, 1024) + peerSub := s.d.peers.SubscribePeerDrops(peerDrop) + defer peerSub.Unsubscribe() + for { // Enable sending of the first buffered element if there is one. var ( @@ -142,6 +149,20 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { finished = append(finished, req) delete(active, pack.PeerId()) + // Handle dropped peer connections: + case p := <-peerDrop: + // Skip if no request is currently pending + req := active[p.id] + if req == nil { + continue + } + // Finalize the request and queue up for processing + req.timer.Stop() + req.dropped = true + + finished = append(finished, req) + delete(active, p.id) + // Handle timed-out requests: case req := <-timeout: // If the peer is already requesting something else, ignore the stale timeout. @@ -166,6 +187,9 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { log.Warn("Busy peer assigned new state fetch", "peer", old.peer.id) // Make sure the previous one doesn't get siletly lost + old.timer.Stop() + old.dropped = true + finished = append(finished, old) } // Start a timer to notify the sync loop if the peer stalled. @@ -187,10 +211,13 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync { type stateSync struct { d *Downloader // Downloader instance to access and manage current peerset - sched *state.StateSync // State trie sync scheduler defining the tasks + sched *trie.TrieSync // State trie sync scheduler defining the tasks keccak hash.Hash // Keccak256 hasher to verify deliveries with tasks map[common.Hash]*stateTask // Set of tasks currently queued for retrieval + numUncommitted int + bytesUncommitted int + deliver chan *stateReq // Delivery channel multiplexing peer responses cancel chan struct{} // Channel to signal a termination request cancelOnce sync.Once // Ensures cancel only ever gets called once @@ -252,9 +279,10 @@ func (s *stateSync) loop() error { // Keep assigning new tasks until the sync completes or aborts for s.sched.Pending() > 0 { - if err := s.assignTasks(); err != nil { + if err := s.commit(false); err != nil { return err } + s.assignTasks() // Tasks assigned, wait for something to happen select { case <-newPeer: @@ -264,9 +292,9 @@ func (s *stateSync) loop() error { return errCancelStateFetch case req := <-s.deliver: - // Response or timeout triggered, drop the peer if stalling - log.Trace("Received node data response", "peer", req.peer.id, "count", len(req.response), "timeout", req.timedOut()) - if len(req.items) <= 2 && req.timedOut() { + // Response, disconnect or timeout triggered, drop the peer if stalling + log.Trace("Received node data response", "peer", req.peer.id, "count", len(req.response), "dropped", req.dropped, "timeout", !req.dropped && req.timedOut()) + if len(req.items) <= 2 && !req.dropped && req.timedOut() { // 2 items are the minimum requested, if even that times out, we've no use of // this peer at the moment. log.Warn("Stalling state sync, dropping peer", "peer", req.peer.id) @@ -284,12 +312,28 @@ func (s *stateSync) loop() error { } } } + return s.commit(true) +} + +func (s *stateSync) commit(force bool) error { + if !force && s.bytesUncommitted < ethdb.IdealBatchSize { + return nil + } + start := time.Now() + b := s.d.stateDB.NewBatch() + s.sched.Commit(b) + if err := b.Write(); err != nil { + return fmt.Errorf("DB write error: %v", err) + } + s.updateStats(s.numUncommitted, 0, 0, time.Since(start)) + s.numUncommitted = 0 + s.bytesUncommitted = 0 return nil } // assignTasks attempts to assing new tasks to all idle peers, either from the // batch currently being retried, or fetching new data from the trie sync itself. -func (s *stateSync) assignTasks() error { +func (s *stateSync) assignTasks() { // Iterate over all idle peers and try to assign them state fetches peers, _ := s.d.peers.NodeDataIdlePeers() for _, p := range peers { @@ -301,7 +345,6 @@ func (s *stateSync) assignTasks() error { // If the peer was assigned tasks to fetch, send the network request if len(req.items) > 0 { req.peer.log.Trace("Requesting new batch of data", "type", "state", "count", len(req.items)) - select { case s.d.trackStateReq <- req: req.peer.FetchNodeData(req.items) @@ -309,7 +352,6 @@ func (s *stateSync) assignTasks() error { } } } - return nil } // fillTasks fills the given request object with a maximum of n state download @@ -347,11 +389,11 @@ func (s *stateSync) fillTasks(n int, req *stateReq) { // delivered. func (s *stateSync) process(req *stateReq) (bool, error) { // Collect processing stats and update progress if valid data was received - processed, written, duplicate, unexpected := 0, 0, 0, 0 + duplicate, unexpected := 0, 0 defer func(start time.Time) { - if processed+written+duplicate+unexpected > 0 { - s.updateStats(processed, written, duplicate, unexpected, time.Since(start)) + if duplicate > 0 || unexpected > 0 { + s.updateStats(0, duplicate, unexpected, time.Since(start)) } }(time.Now()) @@ -362,7 +404,9 @@ func (s *stateSync) process(req *stateReq) (bool, error) { prog, hash, err := s.processNodeData(blob) switch err { case nil: - processed++ + s.numUncommitted++ + s.bytesUncommitted += len(blob) + progress = progress || prog case trie.ErrNotRequested: unexpected++ case trie.ErrAlreadyProcessed: @@ -370,38 +414,20 @@ func (s *stateSync) process(req *stateReq) (bool, error) { default: return stale, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) } - if prog { - progress = true - } // If the node delivered a requested item, mark the delivery non-stale if _, ok := req.tasks[hash]; ok { delete(req.tasks, hash) stale = false } } - // If some data managed to hit the database, flush and reset failure counters - if progress { - // Flush any accumulated data out to disk - batch := s.d.stateDB.NewBatch() - - count, err := s.sched.Commit(batch) - if err != nil { - return stale, err - } - if err := batch.Write(); err != nil { - return stale, err - } - written = count - - // If we're inside the critical section, reset fail counter since we progressed - if atomic.LoadUint32(&s.d.fsPivotFails) > 1 { - log.Trace("Fast-sync progressed, resetting fail counter", "previous", atomic.LoadUint32(&s.d.fsPivotFails)) - atomic.StoreUint32(&s.d.fsPivotFails, 1) // Don't ever reset to 0, as that will unlock the pivot block - } + // If we're inside the critical section, reset fail counter since we progressed. + if progress && atomic.LoadUint32(&s.d.fsPivotFails) > 1 { + log.Trace("Fast-sync progressed, resetting fail counter", "previous", atomic.LoadUint32(&s.d.fsPivotFails)) + atomic.StoreUint32(&s.d.fsPivotFails, 1) // Don't ever reset to 0, as that will unlock the pivot block } + // Put unfulfilled tasks back into the retry queue npeers := s.d.peers.Len() - for hash, task := range req.tasks { // If the node did deliver something, missing items may be due to a protocol // limit or a previous timeout + delayed delivery. Both cases should permit @@ -425,25 +451,25 @@ func (s *stateSync) process(req *stateReq) (bool, error) { // error occurred. func (s *stateSync) processNodeData(blob []byte) (bool, common.Hash, error) { res := trie.SyncResult{Data: blob} - s.keccak.Reset() s.keccak.Write(blob) s.keccak.Sum(res.Hash[:0]) - committed, _, err := s.sched.Process([]trie.SyncResult{res}) return committed, res.Hash, err } // updateStats bumps the various state sync progress counters and displays a log // message for the user to see. -func (s *stateSync) updateStats(processed, written, duplicate, unexpected int, duration time.Duration) { +func (s *stateSync) updateStats(written, duplicate, unexpected int, duration time.Duration) { s.d.syncStatsLock.Lock() defer s.d.syncStatsLock.Unlock() s.d.syncStatsState.pending = uint64(s.sched.Pending()) - s.d.syncStatsState.processed += uint64(processed) + s.d.syncStatsState.processed += uint64(written) s.d.syncStatsState.duplicate += uint64(duplicate) s.d.syncStatsState.unexpected += uint64(unexpected) - log.Info("Imported new state entries", "count", processed, "flushed", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) + if written > 0 || duplicate > 0 || unexpected > 0 { + log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) + } } diff --git a/eth/handler.go b/eth/handler.go index 28ae208c06..cee719ddb6 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -609,7 +609,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Schedule all the unknown hashes for retrieval unknown := make(newBlockHashesData, 0, len(announces)) for _, block := range announces { - if !pm.blockchain.HasBlock(block.Hash) { + if !pm.blockchain.HasBlock(block.Hash, block.Number) { unknown = append(unknown, block) } } @@ -699,7 +699,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { return } // Otherwise if the block is indeed in out own chain, announce it - if pm.blockchain.HasBlock(hash) { + if pm.blockchain.HasBlock(hash, block.NumberU64()) { for _, peer := range peers { peer.SendNewBlockHashes([]common.Hash{hash}, []uint64{block.NumberU64()}) } diff --git a/eth/sync.go b/eth/sync.go index 7442f912c7..a8ae646170 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -188,7 +188,17 @@ func (pm *ProtocolManager) synchronise(peer *peer) { atomic.StoreUint32(&pm.fastSync, 1) mode = downloader.FastSync } - if err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode); err != nil { + // Run the sync cycle, and disable fast sync if we've went past the pivot block + err := pm.downloader.Synchronise(peer.id, pHead, pTd, mode) + + if atomic.LoadUint32(&pm.fastSync) == 1 { + // Disable fast sync if we indeed have something in our chain + if pm.blockchain.CurrentBlock().NumberU64() > 0 { + log.Info("Fast sync complete, auto disabling") + atomic.StoreUint32(&pm.fastSync, 0) + } + } + if err != nil { return } atomic.StoreUint32(&pm.acceptTxs, 1) // Mark initial sync done @@ -201,12 +211,4 @@ func (pm *ProtocolManager) synchronise(peer *peer) { // more reliably update peers or the local TD state. go pm.BroadcastBlock(head, false) } - // If fast sync was enabled, and we synced up, disable it - if atomic.LoadUint32(&pm.fastSync) == 1 { - // Disable fast sync if we indeed have something in our chain - if pm.blockchain.CurrentBlock().NumberU64() > 0 { - log.Info("Fast sync complete, auto disabling") - atomic.StoreUint32(&pm.fastSync, 0) - } - } } diff --git a/ethdb/database.go b/ethdb/database.go index 7d5fb0b9ef..93755dd7e3 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -109,6 +109,10 @@ func (db *LDBDatabase) Put(key []byte, value []byte) error { return db.db.Put(key, value, nil) } +func (db *LDBDatabase) Has(key []byte) (bool, error) { + return db.db.Has(key, nil) +} + // Get returns the given key if it's present. func (db *LDBDatabase) Get(key []byte) ([]byte, error) { // Measure the database get latency, if requested @@ -271,19 +275,19 @@ func (db *LDBDatabase) meter(refresh time.Duration) { } } -// TODO: remove this stuff and expose leveldb directly - func (db *LDBDatabase) NewBatch() Batch { return &ldbBatch{db: db.db, b: new(leveldb.Batch)} } type ldbBatch struct { - db *leveldb.DB - b *leveldb.Batch + db *leveldb.DB + b *leveldb.Batch + size int } func (b *ldbBatch) Put(key, value []byte) error { b.b.Put(key, value) + b.size += len(value) return nil } @@ -291,6 +295,10 @@ func (b *ldbBatch) Write() error { return b.db.Write(b.b, nil) } +func (b *ldbBatch) ValueSize() int { + return b.size +} + type table struct { db Database prefix string @@ -309,6 +317,10 @@ func (dt *table) Put(key []byte, value []byte) error { return dt.db.Put(append([]byte(dt.prefix), key...), value) } +func (dt *table) Has(key []byte) (bool, error) { + return dt.db.Has(append([]byte(dt.prefix), key...)) +} + func (dt *table) Get(key []byte) ([]byte, error) { return dt.db.Get(append([]byte(dt.prefix), key...)) } @@ -342,3 +354,7 @@ func (tb *tableBatch) Put(key, value []byte) error { func (tb *tableBatch) Write() error { return tb.batch.Write() } + +func (tb *tableBatch) ValueSize() int { + return tb.batch.ValueSize() +} diff --git a/ethdb/interface.go b/ethdb/interface.go index f4b787a52a..99a5b770d5 100644 --- a/ethdb/interface.go +++ b/ethdb/interface.go @@ -16,15 +16,29 @@ package ethdb -type Database interface { +// Code using batches should try to add this much data to the batch. +// The value was determined empirically. +const IdealBatchSize = 100 * 1024 + +// Putter wraps the database write operation supported by both batches and regular databases. +type Putter interface { Put(key []byte, value []byte) error +} + +// Database wraps all database operations. All methods are safe for concurrent use. +type Database interface { + Putter Get(key []byte) ([]byte, error) + Has(key []byte) (bool, error) Delete(key []byte) error Close() NewBatch() Batch } +// Batch is a write-only database that commits changes to its host database +// when Write is called. Batch cannot be used concurrently. type Batch interface { - Put(key, value []byte) error + Putter + ValueSize() int // amount of data in the batch Write() error } diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 11b0937248..699bd0c9fd 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -45,6 +45,14 @@ func (db *MemDatabase) Put(key []byte, value []byte) error { return nil } +func (db *MemDatabase) Has(key []byte) (bool, error) { + db.lock.RLock() + defer db.lock.RUnlock() + + _, ok := db.db[string(key)] + return ok, nil +} + func (db *MemDatabase) Get(key []byte) ([]byte, error) { db.lock.RLock() defer db.lock.RUnlock() @@ -93,21 +101,16 @@ type kv struct{ k, v []byte } type memBatch struct { db *MemDatabase writes []kv - lock sync.RWMutex + size int } func (b *memBatch) Put(key, value []byte) error { - b.lock.Lock() - defer b.lock.Unlock() - b.writes = append(b.writes, kv{common.CopyBytes(key), common.CopyBytes(value)}) + b.size += len(value) return nil } func (b *memBatch) Write() error { - b.lock.RLock() - defer b.lock.RUnlock() - b.db.lock.Lock() defer b.db.lock.Unlock() @@ -116,3 +119,7 @@ func (b *memBatch) Write() error { } return nil } + +func (b *memBatch) ValueSize() int { + return b.size +} diff --git a/interfaces.go b/interfaces.go index 744f07b95a..67f236ef75 100644 --- a/interfaces.go +++ b/interfaces.go @@ -102,7 +102,7 @@ type SyncProgress struct { CurrentBlock uint64 // Current block number where sync is at HighestBlock uint64 // Highest alleged block number in the chain PulledStates uint64 // Number of state trie entries already downloaded - KnownStates uint64 // Total number os state trie entries known about + KnownStates uint64 // Total number of state trie entries known about } // ChainSyncReader wraps access to the node's current sync status. If there's no @@ -129,7 +129,7 @@ type ContractCaller interface { CallContract(ctx context.Context, call CallMsg, blockNumber *big.Int) ([]byte, error) } -// FilterQuery contains options for contact log filtering. +// FilterQuery contains options for contract log filtering. type FilterQuery struct { FromBlock *big.Int // beginning of the queried range, nil means genesis block ToBlock *big.Int // end of the range, nil means latest block diff --git a/internal/debug/api.go b/internal/debug/api.go index 8b7693f6a4..3547b05641 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -176,6 +176,17 @@ func (*HandlerT) Stacks() string { return string(buf) } +// FreeOSMemory returns unused memory to the OS. +func (*HandlerT) FreeOSMemory() { + debug.FreeOSMemory() +} + +// SetGCPercent sets the garbage collection target percentage. It returns the previous +// setting. A negative value disables GC. +func (*HandlerT) SetGCPercent(v int) int { + return debug.SetGCPercent(v) +} + func writeProfile(name, file string) error { p := pprof.Lookup(name) log.Info("Writing profile records", "count", p.Count(), "type", name, "dump", file) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 9ec716be20..1ae6e2d738 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -34,58 +34,56 @@ var Modules = map[string]string{ const Chequebook_JS = ` web3._extend({ - property: 'chequebook', - methods: - [ - new web3._extend.Method({ - name: 'deposit', - call: 'chequebook_deposit', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Property({ + property: 'chequebook', + methods: [ + new web3._extend.Method({ + name: 'deposit', + call: 'chequebook_deposit', + params: 1, + inputFormatter: [null] + }), + new web3._extend.Property({ name: 'balance', getter: 'chequebook_balance', - outputFormatter: web3._extend.utils.toDecimal + outputFormatter: web3._extend.utils.toDecimal }), - new web3._extend.Method({ - name: 'cash', - call: 'chequebook_cash', - params: 1, - inputFormatter: [null] - }), - new web3._extend.Method({ - name: 'issue', - call: 'chequebook_issue', - params: 2, - inputFormatter: [null, null] - }), - ] + new web3._extend.Method({ + name: 'cash', + call: 'chequebook_cash', + params: 1, + inputFormatter: [null] + }), + new web3._extend.Method({ + name: 'issue', + call: 'chequebook_issue', + params: 2, + inputFormatter: [null, null] + }), + ] }); ` const Clique_JS = ` web3._extend({ - property: 'clique', - methods: - [ + property: 'clique', + methods: [ new web3._extend.Method({ name: 'getSnapshot', call: 'clique_getSnapshot', params: 1, - inputFormatter: [null] + inputFormatter: [null] }), new web3._extend.Method({ name: 'getSnapshotAtHash', call: 'clique_getSnapshotAtHash', params: 1 }), - new web3._extend.Method({ - name: 'getSigners', - call: 'clique_getSigners', - params: 1, - inputFormatter: [null] - }), + new web3._extend.Method({ + name: 'getSigners', + call: 'clique_getSigners', + params: 1, + inputFormatter: [null] + }), new web3._extend.Method({ name: 'getSignersAtHash', call: 'clique_getSignersAtHash', @@ -100,10 +98,9 @@ web3._extend({ name: 'discard', call: 'clique_discard', params: 1 - }) - ], - properties: - [ + }), + ], + properties: [ new web3._extend.Property({ name: 'proposals', getter: 'clique_proposals' @@ -115,8 +112,7 @@ web3._extend({ const Admin_JS = ` web3._extend({ property: 'admin', - methods: - [ + methods: [ new web3._extend.Method({ name: 'addPeer', call: 'admin_addPeer', @@ -162,10 +158,9 @@ web3._extend({ new web3._extend.Method({ name: 'stopWS', call: 'admin_stopWS' - }) + }), ], - properties: - [ + properties: [ new web3._extend.Property({ name: 'nodeInfo', getter: 'admin_nodeInfo' @@ -177,7 +172,7 @@ web3._extend({ new web3._extend.Property({ name: 'datadir', getter: 'admin_datadir' - }) + }), ] }); ` @@ -185,8 +180,7 @@ web3._extend({ const Debug_JS = ` web3._extend({ property: 'debug', - methods: - [ + methods: [ new web3._extend.Method({ name: 'printBlock', call: 'debug_printBlock', @@ -268,6 +262,16 @@ web3._extend({ params: 0, outputFormatter: console.log }), + new web3._extend.Method({ + name: 'freeOSMemory', + call: 'debug_freeOSMemory', + params: 0, + }), + new web3._extend.Method({ + name: 'setGCPercent', + call: 'debug_setGCPercent', + params: 1, + }), new web3._extend.Method({ name: 'memStats', call: 'debug_memStats', @@ -358,8 +362,7 @@ web3._extend({ const Eth_JS = ` web3._extend({ property: 'eth', - methods: - [ + methods: [ new web3._extend.Method({ name: 'sign', call: 'eth_sign', @@ -396,10 +399,9 @@ web3._extend({ }, params: 2, inputFormatter: [web3._extend.formatters.inputBlockNumberFormatter, web3._extend.utils.toHex] - }) + }), ], - properties: - [ + properties: [ new web3._extend.Property({ name: 'pendingTransactions', getter: 'eth_pendingTransactions', @@ -411,7 +413,7 @@ web3._extend({ } return formatted; } - }) + }), ] }); ` @@ -419,8 +421,7 @@ web3._extend({ const Miner_JS = ` web3._extend({ property: 'miner', - methods: - [ + methods: [ new web3._extend.Method({ name: 'start', call: 'miner_start', @@ -451,7 +452,7 @@ web3._extend({ new web3._extend.Method({ name: 'getHashrate', call: 'miner_getHashrate' - }) + }), ], properties: [] }); @@ -461,12 +462,11 @@ const Net_JS = ` web3._extend({ property: 'net', methods: [], - properties: - [ + properties: [ new web3._extend.Property({ name: 'version', getter: 'net_version' - }) + }), ] }); ` @@ -474,8 +474,7 @@ web3._extend({ const Personal_JS = ` web3._extend({ property: 'personal', - methods: - [ + methods: [ new web3._extend.Method({ name: 'importRawKey', call: 'personal_importRawKey', @@ -501,14 +500,13 @@ web3._extend({ name: 'deriveAccount', call: 'personal_deriveAccount', params: 3 - }) + }), ], - properties: - [ + properties: [ new web3._extend.Property({ name: 'listWallets', getter: 'personal_listWallets' - }) + }), ] }) ` @@ -517,12 +515,11 @@ const RPC_JS = ` web3._extend({ property: 'rpc', methods: [], - properties: - [ + properties: [ new web3._extend.Property({ name: 'modules', getter: 'rpc_modules' - }) + }), ] }); ` @@ -566,7 +563,7 @@ web3._extend({ name: 'listmounts', call: 'swarmfs_listmounts', params: 0 - }) + }), ] }); ` @@ -593,7 +590,7 @@ web3._extend({ status.queued = web3._extend.utils.toDecimal(status.queued); return status; } - }) + }), ] }); ` diff --git a/les/handler.go b/les/handler.go index 1a75cd369d..df7eb6af51 100644 --- a/les/handler.go +++ b/les/handler.go @@ -70,7 +70,7 @@ func errResp(code errCode, format string, v ...interface{}) error { } type BlockChain interface { - HasHeader(hash common.Hash) bool + HasHeader(hash common.Hash, number uint64) bool GetHeader(hash common.Hash, number uint64) *types.Header GetHeaderByHash(hash common.Hash) *types.Header CurrentHeader() *types.Header diff --git a/light/lightchain.go b/light/lightchain.go index df194ecadf..4c877a771a 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -97,9 +97,14 @@ func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus. } if bc.genesisBlock.Hash() == params.MainnetGenesisHash { // add trusted CHT - WriteTrustedCht(bc.chainDb, TrustedCht{Number: 805, Root: common.HexToHash("85e4286fe0a730390245c49de8476977afdae0eb5530b277f62a52b12313d50f")}) + WriteTrustedCht(bc.chainDb, TrustedCht{Number: 1040, Root: common.HexToHash("bb4fb4076cbe6923c8a8ce8f158452bbe19564959313466989fda095a60884ca")}) log.Info("Added trusted CHT for mainnet") } + if bc.genesisBlock.Hash() == params.TestnetGenesisHash { + // add trusted CHT + WriteTrustedCht(bc.chainDb, TrustedCht{Number: 400, Root: common.HexToHash("2a4befa19e4675d939c3dc22dca8c6ae9fcd642be1f04b06bd6e4203cc304660")}) + log.Info("Added trusted CHT for ropsten testnet") + } if err := bc.loadLastState(); err != nil { return nil, err @@ -252,8 +257,8 @@ func (self *LightChain) GetBodyRLP(ctx context.Context, hash common.Hash) (rlp.R // HasBlock checks if a block is fully present in the database or not, caching // it if present. -func (bc *LightChain) HasBlock(hash common.Hash) bool { - blk, _ := bc.GetBlockByHash(NoOdr, hash) +func (bc *LightChain) HasBlock(hash common.Hash, number uint64) bool { + blk, _ := bc.GetBlock(NoOdr, hash, number) return blk != nil } @@ -418,8 +423,8 @@ func (self *LightChain) GetHeaderByHash(hash common.Hash) *types.Header { // HasHeader checks if a block header is present in the database or not, caching // it if present. -func (bc *LightChain) HasHeader(hash common.Hash) bool { - return bc.hc.HasHeader(hash) +func (bc *LightChain) HasHeader(hash common.Hash, number uint64) bool { + return bc.hc.HasHeader(hash, number) } // GetBlockHashesFromHash retrieves a number of block hashes starting at a given diff --git a/miner/agent.go b/miner/agent.go index 855892a07d..e3cebbd2e2 100644 --- a/miner/agent.go +++ b/miner/agent.go @@ -53,7 +53,19 @@ func (self *CpuAgent) Work() chan<- *Work { return self.workCh } func (self *CpuAgent) SetReturnCh(ch chan<- *Result) { self.returnCh = ch } func (self *CpuAgent) Stop() { + if !atomic.CompareAndSwapInt32(&self.isMining, 1, 0) { + return // agent already stopped + } self.stop <- struct{}{} +done: + // Empty work channel + for { + select { + case <-self.workCh: + default: + break done + } + } } func (self *CpuAgent) Start() { @@ -85,17 +97,6 @@ out: break out } } - -done: - // Empty work channel - for { - select { - case <-self.workCh: - default: - break done - } - } - atomic.StoreInt32(&self.isMining, 0) } func (self *CpuAgent) mine(work *Work, stop <-chan struct{}) { diff --git a/miner/worker.go b/miner/worker.go index e1154ac068..bf24970f5c 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -125,8 +125,6 @@ type worker struct { // atomic status counters mining int32 atWork int32 - - fullValidation bool } func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase common.Address, eth Backend, mux *event.TypeMux) *worker { @@ -146,7 +144,6 @@ func newWorker(config *params.ChainConfig, engine consensus.Engine, coinbase com coinbase: coinbase, agents: make(map[Agent]struct{}), unconfirmed: newUnconfirmedBlocks(eth.BlockChain(), miningLogAtDepth), - fullValidation: false, } // Subscribe TxPreEvent for tx pool worker.txSub = eth.TxPool().SubscribeTxPreEvent(worker.txCh) @@ -268,7 +265,7 @@ func (self *worker) update() { self.currentMu.Lock() acc, _ := types.Sender(self.current.signer, ev.Tx) txs := map[common.Address]types.Transactions{acc: {ev.Tx}} - txset := types.NewTransactionsByPriceAndNonce(txs) + txset := types.NewTransactionsByPriceAndNonce(self.current.signer, txs) self.current.commitTransactions(self.mux, txset, self.chain, self.coinbase) self.currentMu.Unlock() @@ -297,57 +294,38 @@ func (self *worker) wait() { block := result.Block work := result.Work - if self.fullValidation { - if _, err := self.chain.InsertChain(types.Blocks{block}); err != nil { - log.Error("Mined invalid block", "err", err) - continue + // Update the block hash in all logs since it is now available and not when the + // receipt/log of individual transactions were created. + for _, r := range work.receipts { + for _, l := range r.Logs { + l.BlockHash = block.Hash() } - go self.mux.Post(core.NewMinedBlockEvent{Block: block}) - } else { - work.state.CommitTo(self.chainDb, self.config.IsEIP158(block.Number())) - stat, err := self.chain.WriteBlock(block) - if err != nil { - log.Error("Failed writing block to chain", "err", err) - continue - } - // update block hash since it is now available and not when the receipt/log of individual transactions were created - for _, r := range work.receipts { - for _, l := range r.Logs { - l.BlockHash = block.Hash() - } - } - for _, log := range work.state.Logs() { - log.BlockHash = block.Hash() - } - - // check if canon block and write transactions - if stat == core.CanonStatTy { - // This puts transactions in a extra db for rpc - core.WriteTxLookupEntries(self.chainDb, block) - // implicit by posting ChainHeadEvent - mustCommitNewWork = false - } - - // broadcast before waiting for validation - go func(block *types.Block, logs []*types.Log, receipts []*types.Receipt) { - self.mux.Post(core.NewMinedBlockEvent{Block: block}) - var ( - events []interface{} - coalescedLogs []*types.Log - ) - events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) - - if stat == core.CanonStatTy { - events = append(events, core.ChainHeadEvent{Block: block}) - coalescedLogs = logs - } - // post blockchain events - self.chain.PostChainEvents(events, coalescedLogs) - if err := core.WriteBlockReceipts(self.chainDb, block.Hash(), block.NumberU64(), receipts); err != nil { - log.Warn("Failed writing block receipts", "err", err) - } - }(block, work.state.Logs(), work.receipts) } + for _, log := range work.state.Logs() { + log.BlockHash = block.Hash() + } + stat, err := self.chain.WriteBlockAndState(block, work.receipts, work.state) + if err != nil { + log.Error("Failed writing block to chain", "err", err) + continue + } + // check if canon block and write transactions + if stat == core.CanonStatTy { + // implicit by posting ChainHeadEvent + mustCommitNewWork = false + } + // Broadcast the block and announce chain insertion event + self.mux.Post(core.NewMinedBlockEvent{Block: block}) + var ( + events []interface{} + logs = work.state.Logs() + ) + events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) + if stat == core.CanonStatTy { + events = append(events, core.ChainHeadEvent{Block: block}) + } + self.chain.PostChainEvents(events, logs) + // Insert the block into the set of pending ones to wait for confirmations self.unconfirmed.Insert(block.NumberU64(), block.Hash()) @@ -471,7 +449,7 @@ func (self *worker) commitNewWork() { log.Error("Failed to fetch pending transactions", "err", err) return } - txs := types.NewTransactionsByPriceAndNonce(pending) + txs := types.NewTransactionsByPriceAndNonce(self.current.signer, pending) work.commitTransactions(self.mux, txs, self.chain, self.coinbase) // compute uncles for the new block. diff --git a/params/bootnodes.go b/params/bootnodes.go index 6c6c02f8cf..82ab6d48c2 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -41,12 +41,14 @@ var TestnetBootnodes = []string{ // Rinkeby test network. var RinkebyBootnodes = []string{ "enode://a24ac7c5484ef4ed0c5eb2d36620ba4e4aa13b8c84684e1b4aab0cebea2ae45cb4d375b77eab56516d34bfbd3c1a833fc51296ff084b770b94fb9028c4d25ccf@52.169.42.101:30303", // IE + "enode://343149e4feefa15d882d9fe4ac7d88f885bd05ebb735e547f12e12080a9fa07c8014ca6fd7f373123488102fe5e34111f8509cf0b7de3f5b44339c9f25e87cb8@52.3.158.184:30303", // INFURA } // RinkebyV5Bootnodes are the enode URLs of the P2P bootstrap nodes running on the // Rinkeby test network for the experimental RLPx v5 topic-discovery network. var RinkebyV5Bootnodes = []string{ "enode://a24ac7c5484ef4ed0c5eb2d36620ba4e4aa13b8c84684e1b4aab0cebea2ae45cb4d375b77eab56516d34bfbd3c1a833fc51296ff084b770b94fb9028c4d25ccf@52.169.42.101:30303?discport=30304", // IE + "enode://343149e4feefa15d882d9fe4ac7d88f885bd05ebb735e547f12e12080a9fa07c8014ca6fd7f373123488102fe5e34111f8509cf0b7de3f5b44339c9f25e87cb8@52.3.158.184:30303?discport=30304", // INFURA } // DiscoveryV5Bootnodes are the enode URLs of the P2P bootstrap nodes for the diff --git a/params/config.go b/params/config.go index f4bb6172ba..5c6f0b6bf7 100644 --- a/params/config.go +++ b/params/config.go @@ -32,45 +32,45 @@ var ( var ( // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(1150000), - DAOForkBlock: big.NewInt(1920000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2463000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2675000), - EIP158Block: big.NewInt(2675000), - MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(1150000), + DAOForkBlock: big.NewInt(1920000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2463000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2675000), + EIP158Block: big.NewInt(2675000), + ByzantiumBlock: big.NewInt(math.MaxInt64), // Don't enable yet Ethash: new(EthashConfig), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. TestnetChainConfig = &ChainConfig{ - ChainId: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet + ChainId: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + ByzantiumBlock: big.NewInt(1700000), Ethash: new(EthashConfig), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. RinkebyChainConfig = &ChainConfig{ - ChainId: big.NewInt(4), - HomesteadBlock: big.NewInt(1), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(2), - EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), - EIP155Block: big.NewInt(3), - EIP158Block: big.NewInt(3), - MetropolisBlock: big.NewInt(math.MaxInt64), // Don't enable yet + ChainId: big.NewInt(4), + HomesteadBlock: big.NewInt(1), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(2), + EIP150Hash: common.HexToHash("0x9b095b36c15eaf13044373aef8ee0bd3a382a5abb92e402afa44b8249c3a90e9"), + EIP155Block: big.NewInt(3), + EIP158Block: big.NewInt(3), + ByzantiumBlock: big.NewInt(math.MaxInt64), // Don't enable yet Clique: &CliqueConfig{ Period: 15, @@ -86,8 +86,8 @@ var ( // means that all fields must be set at all times. This forces // anyone adding flags to the config to also have to set these // fields. - AllProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(math.MaxInt64) /*disabled*/, new(EthashConfig), nil} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} + AllProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -110,7 +110,7 @@ type ChainConfig struct { EIP155Block *big.Int `json:"eip155Block,omitempty"` // EIP155 HF block EIP158Block *big.Int `json:"eip158Block,omitempty"` // EIP158 HF block - MetropolisBlock *big.Int `json:"metropolisBlock,omitempty"` // Metropolis switch block (nil = no fork, 0 = alraedy on homestead) + ByzantiumBlock *big.Int `json:"byzantiumBlock,omitempty"` // Byzantium switch block (nil = no fork, 0 = alraedy on homestead) // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` @@ -147,7 +147,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Metropolis: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Engine: %v}", c.ChainId, c.HomesteadBlock, c.DAOForkBlock, @@ -155,7 +155,7 @@ func (c *ChainConfig) String() string { c.EIP150Block, c.EIP155Block, c.EIP158Block, - c.MetropolisBlock, + c.ByzantiumBlock, engine, ) } @@ -182,8 +182,8 @@ func (c *ChainConfig) IsEIP158(num *big.Int) bool { return isForked(c.EIP158Block, num) } -func (c *ChainConfig) IsMetropolis(num *big.Int) bool { - return isForked(c.MetropolisBlock, num) +func (c *ChainConfig) IsByzantium(num *big.Int) bool { + return isForked(c.ByzantiumBlock, num) } // GasTable returns the gas table corresponding to the current phase (homestead or homestead reprice). @@ -243,8 +243,8 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if c.IsEIP158(head) && !configNumEqual(c.ChainId, newcfg.ChainId) { return newCompatError("EIP158 chain ID", c.EIP158Block, newcfg.EIP158Block) } - if isForkIncompatible(c.MetropolisBlock, newcfg.MetropolisBlock, head) { - return newCompatError("Metropolis fork block", c.MetropolisBlock, newcfg.MetropolisBlock) + if isForkIncompatible(c.ByzantiumBlock, newcfg.ByzantiumBlock, head) { + return newCompatError("Byzantium fork block", c.ByzantiumBlock, newcfg.ByzantiumBlock) } return nil } @@ -312,7 +312,7 @@ func (err *ConfigCompatError) Error() string { type Rules struct { ChainId *big.Int IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool - IsMetropolis bool + IsByzantium bool } func (c *ChainConfig) Rules(num *big.Int) Rules { @@ -320,5 +320,5 @@ func (c *ChainConfig) Rules(num *big.Int) Rules { if chainId == nil { chainId = new(big.Int) } - return Rules{ChainId: new(big.Int).Set(chainId), IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), IsEIP155: c.IsEIP155(num), IsEIP158: c.IsEIP158(num), IsMetropolis: c.IsMetropolis(num)} + return Rules{ChainId: new(big.Int).Set(chainId), IsHomestead: c.IsHomestead(num), IsEIP150: c.IsEIP150(num), IsEIP155: c.IsEIP155(num), IsEIP158: c.IsEIP158(num), IsByzantium: c.IsByzantium(num)} } diff --git a/params/protocol_params.go b/params/protocol_params.go index 9c84c7d348..c56faf56f8 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -64,9 +64,9 @@ const ( Ripemd160PerWordGas uint64 = 120 // Per-word price for a RIPEMD160 operation IdentityBaseGas uint64 = 15 // Base price for a data copy operation IdentityPerWordGas uint64 = 3 // Per-work price for a data copy operation - ModExpQuadCoeffDiv uint64 = 100 // Divisor for the quadratic particle of the big int modular exponentiation + ModExpQuadCoeffDiv uint64 = 20 // Divisor for the quadratic particle of the big int modular exponentiation Bn256AddGas uint64 = 500 // Gas needed for an elliptic curve addition - Bn256ScalarMulGas uint64 = 2000 // Gas needed for an elliptic curve scalar multiplication + Bn256ScalarMulGas uint64 = 40000 // Gas needed for an elliptic curve scalar multiplication Bn256PairingBaseGas uint64 = 100000 // Base price for an elliptic curve pairing check Bn256PairingPerPointGas uint64 = 80000 // Per-point price for an elliptic curve pairing check ) diff --git a/params/version.go b/params/version.go index 0896662606..80329c91d9 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 7 // Minor version component of the current release - VersionPatch = 0 // Patch version component of the current release + VersionPatch = 1 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/swarm/api/api.go b/swarm/api/api.go index 7d185ab3ca..a5941fb5cd 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -132,6 +132,7 @@ func (self *Api) Put(content, contentType string) (storage.Key, error) { func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) { trie, err := loadManifest(self.dpa, key, nil) if err != nil { + status = http.StatusNotFound log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err)) return } diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go new file mode 100644 index 0000000000..ebb5e3ebe2 --- /dev/null +++ b/swarm/api/http/error.go @@ -0,0 +1,116 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +Show nicely (but simple) formatted HTML error pages (or respond with JSON +if the appropriate `Accept` header is set)) for the http package. +*/ +package http + +import ( + "encoding/json" + "fmt" + "html/template" + "net/http" + "time" + + "github.com/ethereum/go-ethereum/log" +) + +//templateMap holds a mapping of an HTTP error code to a template +var templateMap map[int]*template.Template + +//parameters needed for formatting the correct HTML page +type ErrorParams struct { + Msg string + Code int + Timestamp string + template *template.Template + Details template.HTML +} + +//we init the error handling right on boot time, so lookup and http response is fast +func init() { + initErrHandling() +} + +func initErrHandling() { + //pages are saved as strings - get these strings + genErrPage := GetGenericErrorPage() + notFoundPage := GetNotFoundErrorPage() + //map the codes to the available pages + tnames := map[int]string{ + 0: genErrPage, //default + 400: genErrPage, + 404: notFoundPage, + 500: genErrPage, + } + templateMap = make(map[int]*template.Template) + for code, tname := range tnames { + //assign formatted HTML to the code + templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname)) + } +} + +//ShowError is used to show an HTML error page to a client. +//If there is an `Accept` header of `application/json`, JSON will be returned instead +//The function just takes a string message which will be displayed in the error page. +//The code is used to evaluate which template will be displayed +//(and return the correct HTTP status code) +func ShowError(w http.ResponseWriter, r *http.Request, msg string, code int) { + if code == http.StatusInternalServerError { + log.Error(msg) + } + respond(w, r, &ErrorParams{ + Code: code, + Msg: msg, + Timestamp: time.Now().Format(time.RFC1123), + template: getTemplate(code), + }) +} + +//evaluate if client accepts html or json response +func respond(w http.ResponseWriter, r *http.Request, params *ErrorParams) { + w.WriteHeader(params.Code) + if r.Header.Get("Accept") == "application/json" { + respondJson(w, params) + } else { + respondHtml(w, params) + } +} + +//return a HTML page +func respondHtml(w http.ResponseWriter, params *ErrorParams) { + err := params.template.Execute(w, params) + if err != nil { + log.Error(err.Error()) + } +} + +//return JSON +func respondJson(w http.ResponseWriter, params *ErrorParams) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(params) +} + +//get the HTML template for a given code +func getTemplate(code int) *template.Template { + if val, tmpl := templateMap[code]; tmpl { + return val + } else { + return templateMap[0] + } +} diff --git a/swarm/api/http/error_templates.go b/swarm/api/http/error_templates.go new file mode 100644 index 0000000000..29bd3bfbb1 --- /dev/null +++ b/swarm/api/http/error_templates.go @@ -0,0 +1,376 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +We use html templates to handle simple but as informative as possible error pages. + +To eliminate circular dependency in case of an error, we don't store error pages on swarm. +We can't save the error pages as html files on disk, or when deploying compiled binaries +they won't be found. + +For this reason we resort to save the HTML error pages as strings, which then can be +parsed by Go's html/template package +*/ +package http + +//This returns the HTML for generic errors +func GetGenericErrorPage() string { + page := ` + + + + + + + + + + + + Swarm::HTTP Error Page + + + + +
+ +
+
+ +
+
+

There was a problem serving the requested page

+
+
+
{{.Timestamp}}
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + +
+ Hmmmmm....Swarm was not able to serve your request! +
+ Error message: +
+ {{.Msg}} +
+ Error code: +
+ {{.Code}} +
+
+
+ +
+

+ Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution
+ Swarm +

+
+ + +
+ + + +` + return page +} + +//This returns the HTML for a 404 Not Found error +func GetNotFoundErrorPage() string { + page := ` + + + + + + + + + + + + Swarm::404 HTTP Not Found + + + + +
+ +
+
+ +
+
+

Resource Not Found

+
+
+
{{.Timestamp}}
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + +
+ Unfortunately, the resource you were trying to access could not be found on swarm. +
+
+ {{.Msg}} +
+ Error code: +
+ {{.Code}} +
+
+
+ +
+

+ Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution
+ Swarm +

+
+ + +
+ + + +` + return page +} diff --git a/swarm/api/http/error_test.go b/swarm/api/http/error_test.go new file mode 100644 index 0000000000..465fbf4cad --- /dev/null +++ b/swarm/api/http/error_test.go @@ -0,0 +1,143 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package http_test + +import ( + "encoding/json" + "golang.org/x/net/html" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/swarm/testutil" +) + +func TestError(t *testing.T) { + + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/this_should_fail_as_no_bzz_protocol_present" + resp, err := http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 400 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") { + t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode) + } + + _, err = html.Parse(strings.NewReader(string(respbody))) + if err != nil { + t.Fatalf("HTML validation failed for error page returned!") + } +} + +func Test404Page(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/bzz:/1234567890123456789012345678901234567890123456789012345678901234" + resp, err := http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 404 || !strings.Contains(string(respbody), "404") { + t.Fatalf("Invalid Status Code received, expected 404, got %d", resp.StatusCode) + } + + _, err = html.Parse(strings.NewReader(string(respbody))) + if err != nil { + t.Fatalf("HTML validation failed for error page returned!") + } +} + +func Test500Page(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/bzz:/thisShouldFailWith500Code" + resp, err := http.Get(url) + + if err != nil { + t.Fatalf("Request failed: %v", err) + } + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 500 || !strings.Contains(string(respbody), "500") { + t.Fatalf("Invalid Status Code received, expected 500, got %d", resp.StatusCode) + } + + _, err = html.Parse(strings.NewReader(string(respbody))) + if err != nil { + t.Fatalf("HTML validation failed for error page returned!") + } +} + +func TestJsonResponse(t *testing.T) { + srv := testutil.NewTestSwarmServer(t) + defer srv.Close() + + var resp *http.Response + var respbody []byte + + url := srv.URL + "/bzz:/thisShouldFailWith500Code/" + req, err := http.NewRequest("GET", url, nil) + if err != nil { + t.Fatalf("Request failed: %v", err) + } + req.Header.Set("Accept", "application/json") + resp, err = http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Request failed: %v", err) + } + + defer resp.Body.Close() + respbody, err = ioutil.ReadAll(resp.Body) + + if resp.StatusCode != 500 { + t.Fatalf("Invalid Status Code received, expected 500, got %d", resp.StatusCode) + } + + if !isJSON(string(respbody)) { + t.Fatalf("Expected repsonse to be JSON, received invalid JSON: %s", string(respbody)) + } + +} + +func isJSON(s string) bool { + var js map[string]interface{} + return json.Unmarshal([]byte(s), &js) == nil +} diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 0b4ec7e18e..a637b87351 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -332,7 +332,7 @@ func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) { return api.SkipManifest }) if entry == nil { - http.NotFound(w, &r.Request) + s.NotFound(w, r, fmt.Errorf("Manifest entry could not be loaded")) return } key = storage.Key(common.Hex2Bytes(entry.Hash)) @@ -341,8 +341,7 @@ func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size reader := s.api.Retrieve(key) if _, err := reader.Size(nil); err != nil { - s.logDebug("key not found %s: %s", key, err) - http.NotFound(w, &r.Request) + s.NotFound(w, r, fmt.Errorf("Root chunk not found %s: %s", key, err)) return } @@ -534,16 +533,20 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { return } - reader, contentType, _, err := s.api.Get(key, r.uri.Path) + reader, contentType, status, err := s.api.Get(key, r.uri.Path) if err != nil { - s.Error(w, r, err) + switch status { + case http.StatusNotFound: + s.NotFound(w, r, err) + default: + s.Error(w, r, err) + } return } // check the root chunk exists by retrieving the file's size if _, err := reader.Size(nil); err != nil { - s.logDebug("file not found %s: %s", r.uri, err) - http.NotFound(w, &r.Request) + s.NotFound(w, r, fmt.Errorf("File not found %s: %s", r.uri, err)) return } @@ -556,14 +559,14 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) + req := &Request{Request: *r, uri: uri} if err != nil { s.logError("Invalid URI %q: %s", r.URL.Path, err) - http.Error(w, fmt.Sprintf("Invalid bzz URI: %s", err), http.StatusBadRequest) + s.BadRequest(w, req, fmt.Sprintf("Invalid URI %q: %s", r.URL.Path, err)) return } s.logDebug("%s request received for %s", r.Method, uri) - req := &Request{Request: *r, uri: uri} switch r.Method { case "POST": if uri.Raw() { @@ -579,7 +582,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { // strictly a traditional PUT request which replaces content // at a URI, and POST is more ubiquitous) if uri.Raw() { - http.Error(w, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) + ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest) return } else { s.HandlePostFiles(w, req) @@ -587,7 +590,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "DELETE": if uri.Raw() { - http.Error(w, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) + ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest) return } s.HandleDelete(w, req) @@ -611,7 +614,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { s.HandleGetFile(w, req) default: - http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + ShowError(w, r, fmt.Sprintf("Method "+r.Method+" is not supported.", uri), http.StatusMethodNotAllowed) } } @@ -643,11 +646,13 @@ func (s *Server) logError(format string, v ...interface{}) { } func (s *Server) BadRequest(w http.ResponseWriter, r *Request, reason string) { - s.logDebug("bad request %s %s: %s", r.Method, r.uri, reason) - http.Error(w, reason, http.StatusBadRequest) + ShowError(w, &r.Request, fmt.Sprintf("Bad request %s %s: %s", r.Method, r.uri, reason), http.StatusBadRequest) } func (s *Server) Error(w http.ResponseWriter, r *Request, err error) { - s.logError("error serving %s %s: %s", r.Method, r.uri, err) - http.Error(w, err.Error(), http.StatusInternalServerError) + ShowError(w, &r.Request, fmt.Sprintf("Error serving %s %s: %s", r.Method, r.uri, err), http.StatusInternalServerError) +} + +func (s *Server) NotFound(w http.ResponseWriter, r *Request, err error) { + ShowError(w, &r.Request, fmt.Sprintf("NOT FOUND error serving %s %s: %s", r.Method, r.uri, err), http.StatusNotFound) } diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index d3374594b6..ffeaf6e0d8 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -22,6 +22,7 @@ import ( "fmt" "io/ioutil" "net/http" + "strings" "sync" "testing" @@ -110,9 +111,9 @@ func TestBzzrGetPath(t *testing.T) { } nonhashresponses := []string{ - "error resolving name: no DNS to resolve name: \"name\"\n", - "error resolving nonhash: immutable address not a content hash: \"nonhash\"\n", - "error resolving nonhash: no DNS to resolve name: \"nonhash\"\n", + "error resolving name: no DNS to resolve name: "name"", + "error resolving nonhash: immutable address not a content hash: "nonhash"", + "error resolving nonhash: no DNS to resolve name: "nonhash"", } for i, url := range nonhashtests { @@ -129,7 +130,7 @@ func TestBzzrGetPath(t *testing.T) { if err != nil { t.Fatalf("ReadAll failed: %v", err) } - if string(respbody) != nonhashresponses[i] { + if !strings.Contains(string(respbody), nonhashresponses[i]) { t.Fatalf("Non-Hash response body does not match, expected: %v, got: %v", nonhashresponses[i], string(respbody)) } } diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index 69f3cc6157..93f1d4c2fd 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -140,7 +140,7 @@ func compareGeneratedFileWithFileInMount(t *testing.T, files map[string]fileInfo if err != nil { t.Fatalf("Could not readfile %v : %v", fname, err) } - if bytes.Compare(fileContents, finfo.contents) != 0 { + if !bytes.Equal(fileContents, finfo.contents) { t.Fatalf("File %v contents mismatch: %v , %v", fname, fileContents, finfo.contents) } diff --git a/tests/block_test.go b/tests/block_test.go index 56e1e1e8da..669d3ca08a 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -32,8 +32,6 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`^bcTotalDifficultyTest/(lotsOfLeafs|lotsOfBranches|sideChainWithMoreTransactions)`) // Constantinople is not implemented yet. bt.skipLoad(`(?i)(constantinople)`) - // Expected failures: - bt.fails(`^TransitionTests/bcHomesteadToDao/DaoTransactions(|_UncleExtradata|_EmptyTransactionAndForkBlocksAhead)\.json`, "issue in test") // Still failing tests bt.skipLoad(`^bcWalletTest.*_Byzantium$`) diff --git a/tests/init.go b/tests/init.go index 0c3fe61d13..a2c633ad67 100644 --- a/tests/init.go +++ b/tests/init.go @@ -45,13 +45,13 @@ var Forks = map[string]*params.ChainConfig{ EIP158Block: big.NewInt(0), }, "Byzantium": ¶ms.ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - DAOForkBlock: big.NewInt(0), - MetropolisBlock: big.NewInt(0), + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + DAOForkBlock: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), }, "FrontierToHomesteadAt5": ¶ms.ChainConfig{ ChainId: big.NewInt(1), @@ -69,12 +69,12 @@ var Forks = map[string]*params.ChainConfig{ DAOForkSupport: true, }, "EIP158ToByzantiumAt5": ¶ms.ChainConfig{ - ChainId: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - MetropolisBlock: big.NewInt(5), + ChainId: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(5), }, } diff --git a/tests/state_test.go b/tests/state_test.go index 1fb7f5908f..9a7430fbeb 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -39,6 +39,13 @@ func TestState(t *testing.T) { st.fails(`^stRevertTest/RevertPrefoundEmptyOOG\.json/EIP158`, "bug in test") st.fails(`^stRevertTest/RevertPrecompiledTouch\.json/Byzantium`, "bug in test") st.fails(`^stRevertTest/RevertPrefoundEmptyOOG\.json/Byzantium`, "bug in test") + st.fails( `^stRandom/randomStatetest645\.json/EIP150/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest645\.json/Frontier/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest645\.json/Homestead/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest644\.json/EIP150/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest644\.json/Frontier/.*`, "known bug #15119") + st.fails( `^stRandom/randomStatetest644\.json/Homestead/.*`, "known bug #15119") + st.walk(t, stateTestDir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { @@ -59,7 +66,8 @@ func TestState(t *testing.T) { } // Transactions with gasLimit above this value will not get a VM trace on failure. -const traceErrorLimit = 400000 +//const traceErrorLimit = 400000 +const traceErrorLimit = 0 func withTrace(t *testing.T, gasLimit uint64, test func(vm.Config) error) { err := test(vm.Config{}) diff --git a/tests/testdata b/tests/testdata index 1d30b47956..ca41e90635 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 1d30b4795664f64b1b157971754e14a10cfd9115 +Subproject commit ca41e906351209481bce3a1b35501f25a79023c5 diff --git a/tests/transaction_test.go b/tests/transaction_test.go index 72d43c0ece..c743996c2a 100644 --- a/tests/transaction_test.go +++ b/tests/transaction_test.go @@ -37,12 +37,12 @@ func TestTransaction(t *testing.T) { EIP158Block: big.NewInt(0), ChainId: big.NewInt(1), }) - txt.config(`^Metropolis/`, params.ChainConfig{ - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - MetropolisBlock: big.NewInt(0), + txt.config(`^Byzantium/`, params.ChainConfig{ + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), }) txt.walk(t, transactionTestDir, func(t *testing.T, name string, test *TransactionTest) { diff --git a/trie/hasher.go b/trie/hasher.go index 672456e2d9..4719aabf62 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -76,7 +76,7 @@ func (h *hasher) hash(n node, db DatabaseWriter, force bool) (node, node, error) if err != nil { return hashNode{}, n, err } - // Cache the hash of the ndoe for later reuse and remove + // Cache the hash of the node for later reuse and remove // the dirty flag in commit mode. It's fine to assign these values directly // without copying the node first because hashChildren copies it. cachedHash, _ := hashed.(hashNode) diff --git a/trie/sync.go b/trie/sync.go index 1e4f8d87c1..fea10051f4 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -138,7 +138,7 @@ func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) if _, ok := s.membatch.batch[hash]; ok { return } - if blob, _ := s.database.Get(hash.Bytes()); blob != nil { + if ok, _ := s.database.Has(hash.Bytes()); ok { return } // Assemble the new sub-trie sync request @@ -296,8 +296,7 @@ func (s *TrieSync) children(req *request, object node) ([]*request, error) { if _, ok := s.membatch.batch[hash]; ok { continue } - blob, _ := s.database.Get(node) - if local, err := decodeNode(node[:], blob, 0); local != nil && err == nil { + if ok, _ := s.database.Has(node); ok { continue } // Locally unknown node, schedule for retrieval diff --git a/trie/trie.go b/trie/trie.go index a3151b1ce7..7f69a3d1de 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -66,6 +66,7 @@ type Database interface { // DatabaseReader wraps the Get method of a backing store for the trie. type DatabaseReader interface { Get(key []byte) (value []byte, err error) + Has(key []byte) (bool, error) } // DatabaseWriter wraps the Put method of a backing store for the trie.